This commit is contained in:
Athou
2024-08-07 08:10:14 +02:00
parent 2f6ddf0e70
commit cc32f8ad16
164 changed files with 2011 additions and 3288 deletions

View File

@@ -21,21 +21,21 @@ import org.apache.hc.client5.http.protocol.RedirectLocations;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.eclipse.jetty.http.HttpStatus;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedVersion;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import io.dropwizard.util.DataSize;
import jakarta.inject.Inject;
import io.quarkus.runtime.configuration.MemorySize;
import jakarta.inject.Singleton;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
@@ -51,15 +51,15 @@ import nl.altindag.ssl.apache5.util.Apache5SslUtils;
public class HttpGetter {
private final CloseableHttpClient client;
private final DataSize maxResponseSize;
private final MemorySize maxResponseSize;
@Inject
public HttpGetter(CommaFeedConfiguration config, MetricRegistry metrics) {
PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config.getApplicationSettings().getBackgroundThreads());
String userAgent = Optional.ofNullable(config.getApplicationSettings().getUserAgent())
.orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", config.getVersion()));
public HttpGetter(CommaFeedConfiguration config, CommaFeedVersion version, MetricRegistry metrics) {
PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config.feedRefresh().httpThreads());
String userAgent = config.feedRefresh()
.userAgent()
.orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", version.getVersion()));
this.client = newClient(connectionManager, userAgent);
this.maxResponseSize = config.getApplicationSettings().getMaxFeedResponseSize();
this.maxResponseSize = config.feedRefresh().maxResponseSize();
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "max"), () -> connectionManager.getTotalStats().getMax());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "size"),
@@ -98,7 +98,7 @@ public class HttpGetter {
context.setRequestConfig(RequestConfig.custom().setResponseTimeout(timeout, TimeUnit.MILLISECONDS).build());
HttpResponse response = client.execute(request, context, resp -> {
byte[] content = resp.getEntity() == null ? null : toByteArray(resp.getEntity(), maxResponseSize.toBytes());
byte[] content = resp.getEntity() == null ? null : toByteArray(resp.getEntity(), maxResponseSize.asLongValue());
int code = resp.getCode();
String lastModifiedHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
.map(NameValuePair::getValue)
@@ -120,7 +120,7 @@ public class HttpGetter {
});
int code = response.getCode();
if (code == HttpStatus.NOT_MODIFIED_304) {
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
} else if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);

View File

@@ -1,51 +0,0 @@
package com.commafeed.backend.cache;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import redis.clients.jedis.DefaultJedisClientConfig;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisClientConfig;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.Protocol;
@Getter
public class RedisPoolFactory {
@JsonProperty
private String host = "localhost";
@JsonProperty
private int port = Protocol.DEFAULT_PORT;
@JsonProperty
private String username;
@JsonProperty
private String password;
@JsonProperty
private int timeout = Protocol.DEFAULT_TIMEOUT;
@JsonProperty
private int database = Protocol.DEFAULT_DATABASE;
@JsonProperty
private int maxTotal = 500;
public JedisPool build() {
JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxTotal(maxTotal);
JedisClientConfig clientConfig = DefaultJedisClientConfig.builder()
.user(username)
.password(password)
.timeoutMillis(timeout)
.database(database)
.build();
return new JedisPool(poolConfig, new HostAndPort(host, port), clientConfig);
}
}

View File

@@ -3,25 +3,22 @@ package com.commafeed.backend.dao;
import java.util.List;
import java.util.Objects;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.QFeedCategory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import com.querydsl.core.types.Predicate;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class FeedCategoryDAO extends GenericDAO<FeedCategory> {
private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory;
@Inject
public FeedCategoryDAO(SessionFactory sessionFactory) {
super(sessionFactory);
public FeedCategoryDAO(EntityManager entityManager) {
super(entityManager, FeedCategory.class);
}
public List<FeedCategory> findAll(User user) {

View File

@@ -4,7 +4,6 @@ import java.time.Instant;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed;
@@ -12,8 +11,8 @@ import com.commafeed.backend.model.QFeedSubscription;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.impl.JPAQuery;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class FeedDAO extends GenericDAO<Feed> {
@@ -21,9 +20,8 @@ public class FeedDAO extends GenericDAO<Feed> {
private static final QFeed FEED = QFeed.feed;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
@Inject
public FeedDAO(SessionFactory sessionFactory) {
super(sessionFactory);
public FeedDAO(EntityManager entityManager) {
super(entityManager, Feed.class);
}
public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {

View File

@@ -2,16 +2,14 @@ package com.commafeed.backend.dao;
import java.util.List;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.JPQLSubQuery;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
@@ -19,9 +17,8 @@ public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
@Inject
public FeedEntryContentDAO(SessionFactory sessionFactory) {
super(sessionFactory);
public FeedEntryContentDAO(EntityManager entityManager) {
super(entityManager, FeedEntryContent.class);
}
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {

View File

@@ -3,16 +3,14 @@ package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import lombok.AllArgsConstructor;
import lombok.Getter;
@@ -21,9 +19,8 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
@Inject
public FeedEntryDAO(SessionFactory sessionFactory) {
super(sessionFactory);
public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class);
}
public FeedEntry findExisting(String guidHash, Feed feed) {

View File

@@ -7,7 +7,6 @@ import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.hibernate.SessionFactory;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.feed.FeedEntryKeyword;
@@ -28,8 +27,8 @@ import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.Tuple;
import com.querydsl.jpa.impl.JPAQuery;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
@@ -42,9 +41,8 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config;
@Inject
public FeedEntryStatusDAO(SessionFactory sessionFactory, FeedEntryTagDAO feedEntryTagDAO, CommaFeedConfiguration config) {
super(sessionFactory);
public FeedEntryStatusDAO(EntityManager entityManager, FeedEntryTagDAO feedEntryTagDAO, CommaFeedConfiguration config) {
super(entityManager, FeedEntryStatus.class);
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
@@ -60,8 +58,8 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
*/
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Instant unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
boolean read = unreadThreshold != null && entry.getPublished().isBefore(unreadThreshold);
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
boolean read = statusesInstantThreshold != null && entry.getPublished().isBefore(statusesInstantThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
@@ -84,6 +82,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
boolean includeContent) {
JPAQuery<FeedEntryStatus> query = query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue());
if (includeContent) {
query.join(STATUS.entry).fetchJoin();
query.join(STATUS.entry.content).fetchJoin();
}
@@ -105,7 +104,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
query.limit(limit);
}
setTimeout(query, config.getApplicationSettings().getQueryTimeout());
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = query.fetch();
statuses.forEach(s -> s.setMarkable(true));
@@ -179,7 +178,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
query.limit(limit);
}
setTimeout(query, config.getApplicationSettings().getQueryTimeout());
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = new ArrayList<>();
List<Tuple> tuples = query.fetch();
@@ -217,9 +216,9 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
or.or(STATUS.read.isNull());
or.or(STATUS.read.isFalse());
Instant unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
if (unreadThreshold != null) {
return or.and(ENTRY.published.goe(unreadThreshold));
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
if (statusesInstantThreshold != null) {
return or.and(ENTRY.published.goe(statusesInstantThreshold));
} else {
return or;
}

View File

@@ -4,24 +4,21 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> {
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
@Inject
public FeedEntryTagDAO(SessionFactory sessionFactory) {
super(sessionFactory);
public FeedEntryTagDAO(EntityManager entityManager) {
super(entityManager, FeedEntryTag.class);
}
public List<String> findByUser(User user) {

View File

@@ -5,12 +5,11 @@ import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.hibernate.SessionFactory;
import org.hibernate.engine.spi.SessionFactoryImplementor;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.event.service.spi.EventListenerRegistry;
import org.hibernate.event.spi.EventType;
import org.hibernate.event.spi.PostCommitInsertEventListener;
import org.hibernate.event.spi.PostInsertEvent;
import org.hibernate.event.spi.PostInsertEventListener;
import org.hibernate.persister.entity.EntityPersister;
import com.commafeed.backend.model.AbstractModel;
@@ -23,28 +22,28 @@ import com.commafeed.backend.model.User;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.JPQLQuery;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private final SessionFactory sessionFactory;
private final EntityManager entityManager;
@Inject
public FeedSubscriptionDAO(SessionFactory sessionFactory) {
super(sessionFactory);
this.sessionFactory = sessionFactory;
public FeedSubscriptionDAO(EntityManager entityManager) {
super(entityManager, FeedSubscription.class);
this.entityManager = entityManager;
}
public void onPostCommitInsert(Consumer<FeedSubscription> consumer) {
sessionFactory.unwrap(SessionFactoryImplementor.class)
entityManager.unwrap(SharedSessionContractImplementor.class)
.getFactory()
.getServiceRegistry()
.getService(EventListenerRegistry.class)
.getEventListenerGroup(EventType.POST_COMMIT_INSERT)
.appendListener(new PostInsertEventListener() {
.appendListener(new PostCommitInsertEventListener() {
@Override
public void onPostInsert(PostInsertEvent event) {
if (event.getEntity() instanceof FeedSubscription s) {
@@ -56,6 +55,11 @@ public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
public boolean requiresPostCommitHandling(EntityPersister persister) {
return true;
}
@Override
public void onPostInsertCommitFailed(PostInsertEvent event) {
// do nothing
}
});
}

View File

@@ -2,7 +2,7 @@ package com.commafeed.backend.dao;
import java.util.Collection;
import org.hibernate.SessionFactory;
import org.hibernate.Session;
import org.hibernate.jpa.SpecHints;
import com.commafeed.backend.model.AbstractModel;
@@ -12,45 +12,43 @@ import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import com.querydsl.jpa.impl.JPAUpdateClause;
import io.dropwizard.hibernate.AbstractDAO;
import jakarta.persistence.EntityManager;
import lombok.RequiredArgsConstructor;
public abstract class GenericDAO<T extends AbstractModel> extends AbstractDAO<T> {
@RequiredArgsConstructor
public abstract class GenericDAO<T extends AbstractModel> {
protected GenericDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
private final EntityManager entityManager;
private final Class<T> entityClass;
protected JPAQueryFactory query() {
return new JPAQueryFactory(currentSession());
return new JPAQueryFactory(entityManager);
}
protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) {
return new JPAUpdateClause(currentSession(), entityPath);
return new JPAUpdateClause(entityManager, entityPath);
}
protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(currentSession(), entityPath);
return new JPADeleteClause(entityManager, entityPath);
}
@SuppressWarnings("deprecation")
public void saveOrUpdate(T model) {
persist(model);
entityManager.unwrap(Session.class).saveOrUpdate(model);
}
public void saveOrUpdate(Collection<T> models) {
models.forEach(this::persist);
}
public void update(T model) {
currentSession().merge(model);
models.forEach(this::saveOrUpdate);
}
public T findById(Long id) {
return get(id);
return entityManager.find(entityClass, id);
}
public void delete(T object) {
if (object != null) {
currentSession().remove(object);
entityManager.remove(object);
}
}

View File

@@ -1,68 +1,20 @@
package com.commafeed.backend.dao;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import org.hibernate.Transaction;
import org.hibernate.context.internal.ManagedSessionContext;
import jakarta.inject.Inject;
import io.quarkus.narayana.jta.QuarkusTransaction;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class UnitOfWork {
private final SessionFactory sessionFactory;
public void run(SessionRunner sessionRunner) {
public void run(SessionRunner runner) {
call(() -> {
sessionRunner.runInSession();
runner.runInSession();
return null;
});
}
public <T> T call(SessionRunnerReturningValue<T> sessionRunner) {
T t = null;
boolean sessionAlreadyBound = ManagedSessionContext.hasBind(sessionFactory);
try (Session session = sessionFactory.openSession()) {
if (!sessionAlreadyBound) {
ManagedSessionContext.bind(session);
}
Transaction tx = session.beginTransaction();
try {
t = sessionRunner.runInSession();
commitTransaction(tx);
} catch (Exception e) {
rollbackTransaction(tx);
UnitOfWork.rethrow(e);
}
} finally {
if (!sessionAlreadyBound) {
ManagedSessionContext.unbind(sessionFactory);
}
}
return t;
}
private static void rollbackTransaction(Transaction tx) {
if (tx != null && tx.isActive()) {
tx.rollback();
}
}
private static void commitTransaction(Transaction tx) {
if (tx != null && tx.isActive()) {
tx.commit();
}
}
@SuppressWarnings("unchecked")
private static <E extends Exception> void rethrow(Exception e) throws E {
throw (E) e;
public <T> T call(SessionRunnerReturningValue<T> runner) {
return QuarkusTransaction.joiningExisting().call(runner::runInSession);
}
@FunctionalInterface

View File

@@ -1,21 +1,18 @@
package com.commafeed.backend.dao;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class UserDAO extends GenericDAO<User> {
private static final QUser USER = QUser.user;
@Inject
public UserDAO(SessionFactory sessionFactory) {
super(sessionFactory);
public UserDAO(EntityManager entityManager) {
super(entityManager, User.class);
}
public User findByName(String name) {

View File

@@ -4,24 +4,21 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class UserRoleDAO extends GenericDAO<UserRole> {
private static final QUserRole ROLE = QUserRole.userRole;
@Inject
public UserRoleDAO(SessionFactory sessionFactory) {
super(sessionFactory);
public UserRoleDAO(EntityManager entityManager) {
super(entityManager, UserRole.class);
}
public List<UserRole> findAll() {

View File

@@ -1,22 +1,19 @@
package com.commafeed.backend.dao;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.QUserSettings;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class UserSettingsDAO extends GenericDAO<UserSettings> {
private static final QUserSettings SETTINGS = QUserSettings.userSettings;
@Inject
public UserSettingsDAO(SessionFactory sessionFactory) {
super(sessionFactory);
public UserSettingsDAO(EntityManager entityManager) {
super(entityManager, UserSettings.class);
}
public UserSettings findByUser(User user) {

View File

@@ -10,7 +10,7 @@ import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import jakarta.inject.Inject;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -20,8 +20,9 @@ import lombok.extern.slf4j.Slf4j;
*
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
@Priority(Integer.MIN_VALUE)
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;

View File

@@ -11,13 +11,12 @@ import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {

View File

@@ -22,13 +22,12 @@ import com.google.api.services.youtube.model.ChannelListResponse;
import com.google.api.services.youtube.model.PlaylistListResponse;
import com.google.api.services.youtube.model.Thumbnail;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
@@ -43,8 +42,8 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
return null;
}
String googleAuthKey = config.getApplicationSettings().getGoogleAuthKey();
if (googleAuthKey == null) {
Optional<String> googleAuthKey = config.googleAuthKey();
if (googleAuthKey.isEmpty()) {
log.debug("no google auth key configured");
return null;
}
@@ -63,13 +62,13 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
ChannelListResponse response = null;
if (userId.isPresent()) {
log.debug("contacting youtube api for user {}", userId.get().getValue());
response = fetchForUser(youtube, googleAuthKey, userId.get().getValue());
response = fetchForUser(youtube, googleAuthKey.get(), userId.get().getValue());
} else if (channelId.isPresent()) {
log.debug("contacting youtube api for channel {}", channelId.get().getValue());
response = fetchForChannel(youtube, googleAuthKey, channelId.get().getValue());
response = fetchForChannel(youtube, googleAuthKey.get(), channelId.get().getValue());
} else if (playlistId.isPresent()) {
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
response = fetchForPlaylist(youtube, googleAuthKey, playlistId.get().getValue());
response = fetchForPlaylist(youtube, googleAuthKey.get(), playlistId.get().getValue());
}
if (response == null || response.isEmpty() || CollectionUtils.isEmpty(response.getItems())) {

View File

@@ -2,7 +2,7 @@ package com.commafeed.backend.feed;
import java.io.IOException;
import java.time.Instant;
import java.util.Set;
import java.util.List;
import org.apache.commons.codec.binary.StringUtils;
@@ -15,22 +15,26 @@ import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import com.rometools.rome.io.FeedException;
import jakarta.inject.Inject;
import io.quarkus.arc.All;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches a feed then parses it
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final Set<FeedURLProvider> urlProviders;
private final List<FeedURLProvider> urlProviders;
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedException, IOException, NotModifiedException {
@@ -87,7 +91,7 @@ public class FeedFetcher {
result.getDuration());
}
private static String extractFeedUrl(Set<FeedURLProvider> urlProviders, String url, String urlContent) {
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
for (FeedURLProvider urlProvider : urlProviders) {
String feedUrl = urlProvider.get(url, urlContent);
if (feedUrl != null) {

View File

@@ -1,6 +1,5 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
@@ -21,14 +20,12 @@ import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import io.dropwizard.lifecycle.Managed;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshEngine implements Managed {
public class FeedRefreshEngine {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
@@ -45,7 +42,6 @@ public class FeedRefreshEngine implements Managed {
private final ThreadPoolExecutor workerExecutor;
private final ThreadPoolExecutor databaseUpdaterExecutor;
@Inject
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
CommaFeedConfiguration config, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
@@ -60,15 +56,14 @@ public class FeedRefreshEngine implements Managed {
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.getApplicationSettings().getBackgroundThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.getApplicationSettings().getDatabaseUpdateThreads());
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) workerExecutor::getActiveCount);
metrics.register(MetricRegistry.name(getClass(), "updater", "active"), (Gauge<Integer>) databaseUpdaterExecutor::getActiveCount);
}
@Override
public void start() {
startFeedProcessingLoop();
startRefillLoop();
@@ -165,22 +160,20 @@ public class FeedRefreshEngine implements Managed {
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> {
Instant lastLoginThreshold = Boolean.TRUE.equals(config.getApplicationSettings().getHeavyLoad())
? Instant.now().minus(Duration.ofDays(30))
: null;
Instant lastLoginThreshold = config.feedRefresh().userInactivityPeriod().isZero() ? null
: Instant.now().minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Instant nextUpdateDate = Instant.now().plus(Duration.ofMinutes(config.getApplicationSettings().getRefreshIntervalMinutes()));
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
return feeds;
});
}
private int getBatchSize() {
return Math.min(100, 3 * config.getApplicationSettings().getBackgroundThreads());
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
@Override
public void stop() {
this.feedProcessingLoopExecutor.shutdownNow();
this.refillLoopExecutor.shutdownNow();

View File

@@ -6,24 +6,22 @@ import java.time.temporal.ChronoUnit;
import com.commafeed.CommaFeedConfiguration;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
@Singleton
public class FeedRefreshIntervalCalculator {
private final boolean heavyLoad;
private final int refreshIntervalMinutes;
private final Duration refreshInterval;
private final boolean empiricalInterval;
@Inject
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config) {
this.heavyLoad = config.getApplicationSettings().getHeavyLoad();
this.refreshIntervalMinutes = config.getApplicationSettings().getRefreshIntervalMinutes();
this.refreshInterval = config.feedRefresh().interval();
this.empiricalInterval = config.feedRefresh().intervalEmpirical();
}
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval) {
Instant defaultRefreshInterval = getDefaultRefreshInterval();
return heavyLoad ? computeRefreshIntervalForHeavyLoad(publishedDate, averageEntryInterval, defaultRefreshInterval)
return empiricalInterval ? computeRefreshIntervalForHeavyLoad(publishedDate, averageEntryInterval, defaultRefreshInterval)
: defaultRefreshInterval;
}
@@ -33,7 +31,7 @@ public class FeedRefreshIntervalCalculator {
public Instant onFetchError(int errorCount) {
int retriesBeforeDisable = 3;
if (errorCount < retriesBeforeDisable || !heavyLoad) {
if (errorCount < retriesBeforeDisable || !empiricalInterval) {
return getDefaultRefreshInterval();
}
@@ -42,7 +40,7 @@ public class FeedRefreshIntervalCalculator {
}
private Instant getDefaultRefreshInterval() {
return Instant.now().plus(Duration.ofMinutes(refreshIntervalMinutes));
return Instant.now().plus(refreshInterval);
}
private Instant computeRefreshIntervalForHeavyLoad(Instant publishedDate, Long averageEntryInterval, Instant defaultRefreshInterval) {

View File

@@ -32,7 +32,6 @@ import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import com.google.common.util.concurrent.Striped;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -58,7 +57,6 @@ public class FeedRefreshUpdater {
private final Meter feedUpdated;
private final Meter entryInserted;
@Inject
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO, CacheService cache, WebSocketSessions webSocketSessions) {
this.unitOfWork = unitOfWork;

View File

@@ -16,7 +16,6 @@ import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
@@ -32,7 +31,6 @@ public class FeedRefreshWorker {
private final CommaFeedConfiguration config;
private final Meter feedFetched;
@Inject
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
@@ -51,14 +49,14 @@ public class FeedRefreshWorker {
List<Entry> entries = result.feed().entries();
Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
Integer maxEntriesAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays();
if (maxEntriesAgeDays > 0) {
Instant threshold = Instant.now().minus(Duration.ofDays(maxEntriesAgeDays));
Duration maxEntriesAgeDays = config.database().cleanup().entriesMaxAge();
if (!maxEntriesAgeDays.isZero()) {
Instant threshold = Instant.now().minus(maxEntriesAgeDays);
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList();
}

View File

@@ -38,14 +38,13 @@ import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedInput;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
/**
* Parses raw xml into a FeedParserResult object
*/
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class FeedParser {

View File

@@ -15,11 +15,10 @@ import com.rometools.opml.feed.opml.Attribute;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class OPMLExporter {
@@ -28,7 +27,7 @@ public class OPMLExporter {
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.1");
opml.setFeedType("opml_1.0");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());

View File

@@ -17,13 +17,12 @@ import com.rometools.opml.feed.opml.Outline;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.WireFeedInput;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class OPMLImporter {

View File

@@ -4,10 +4,13 @@ import org.jdom2.Element;
import com.rometools.opml.feed.opml.Opml;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Add missing title to the generated OPML
*
*/
@RegisterForReflection
public class OPML11Generator extends com.rometools.opml.io.impl.OPML10Generator {
public OPML11Generator() {

View File

@@ -9,10 +9,13 @@ import com.rometools.opml.io.impl.OPML10Parser;
import com.rometools.rome.feed.WireFeed;
import com.rometools.rome.io.FeedException;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support for OPML 1.1 parsing
*
*/
@RegisterForReflection
public class OPML11Parser extends OPML10Parser {
public OPML11Parser() {

View File

@@ -6,10 +6,13 @@ import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.impl.ConverterForRSS090;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support description tag for RSS09
*
*/
@RegisterForReflection
public class RSS090DescriptionConverter extends ConverterForRSS090 {
@Override

View File

@@ -8,10 +8,13 @@ import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.io.impl.RSS090Parser;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support description tag for RSS09
*
*/
@RegisterForReflection
public class RSS090DescriptionParser extends RSS090Parser {
@Override

View File

@@ -10,6 +10,9 @@ import org.jdom2.Namespace;
import com.google.common.collect.Lists;
import com.rometools.rome.io.impl.RSS10Parser;
import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection
public class RSSRDF10Parser extends RSS10Parser {
private static final String RSS_URI = "http://purl.org/rss/1.0/";

View File

@@ -21,12 +21,11 @@ import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Slf4j
@Singleton
public class FeedEntryContentCleaningService {

View File

@@ -12,11 +12,10 @@ import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class FeedEntryContentService {

View File

@@ -25,11 +25,10 @@ import org.jsoup.Jsoup;
import com.commafeed.backend.model.FeedEntry;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class FeedEntryFilteringService {

View File

@@ -18,13 +18,12 @@ import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class FeedEntryService {

View File

@@ -10,11 +10,10 @@ import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class FeedEntryTagService {

View File

@@ -2,8 +2,8 @@ package com.commafeed.backend.service;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedDAO;
@@ -14,19 +14,18 @@ import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models;
import com.google.common.io.Resources;
import jakarta.inject.Inject;
import io.quarkus.arc.All;
import jakarta.inject.Singleton;
@Singleton
public class FeedService {
private final FeedDAO feedDAO;
private final Set<AbstractFaviconFetcher> faviconFetchers;
private final List<AbstractFaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
@Inject
public FeedService(FeedDAO feedDAO, Set<AbstractFaviconFetcher> faviconFetchers) {
public FeedService(FeedDAO feedDAO, @All List<AbstractFaviconFetcher> faviconFetchers) {
this.feedDAO = feedDAO;
this.faviconFetchers = faviconFetchers;

View File

@@ -21,7 +21,6 @@ import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
@@ -37,7 +36,6 @@ public class FeedSubscriptionService {
private final CacheService cache;
private final CommaFeedConfiguration config;
@Inject
public FeedSubscriptionService(FeedDAO feedDAO, FeedEntryStatusDAO feedEntryStatusDAO, FeedSubscriptionDAO feedSubscriptionDAO,
FeedService feedService, FeedRefreshEngine feedRefreshEngine, CacheService cache, CommaFeedConfiguration config) {
this.feedDAO = feedDAO;
@@ -63,7 +61,7 @@ public class FeedSubscriptionService {
public long subscribe(User user, String url, String title, FeedCategory category, int position) {
final String pubUrl = config.getApplicationSettings().getPublicUrl();
final String pubUrl = config.publicUrl();
if (StringUtils.isBlank(pubUrl)) {
throw new FeedSubscriptionException("Public URL of this CommaFeed instance is not set");
}
@@ -71,7 +69,7 @@ public class FeedSubscriptionService {
throw new FeedSubscriptionException("Could not subscribe to a feed from this CommaFeed instance");
}
Integer maxFeedsPerUser = config.getApplicationSettings().getMaxFeedsPerUser();
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message = String.format("You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);

View File

@@ -4,10 +4,9 @@ import java.util.Optional;
import java.util.Properties;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.ApplicationSettings;
import com.commafeed.CommaFeedConfiguration.Smtp;
import com.commafeed.backend.model.User;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.mail.Authenticator;
import jakarta.mail.Message;
@@ -22,27 +21,29 @@ import lombok.RequiredArgsConstructor;
* Mailing service
*
*/
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class MailService {
private final CommaFeedConfiguration config;
public void sendMail(User user, String subject, String content) throws Exception {
Optional<Smtp> settings = config.smtp();
if (settings.isEmpty()) {
throw new IllegalArgumentException("SMTP settings not configured");
}
ApplicationSettings settings = config.getApplicationSettings();
final String username = settings.getSmtpUserName();
final String password = settings.getSmtpPassword();
final String fromAddress = Optional.ofNullable(settings.getSmtpFromAddress()).orElse(settings.getSmtpUserName());
final String username = settings.get().userName();
final String password = settings.get().password();
final String fromAddress = Optional.ofNullable(settings.get().fromAddress()).orElse(settings.get().userName());
String dest = user.getEmail();
Properties props = new Properties();
props.put("mail.smtp.auth", "true");
props.put("mail.smtp.starttls.enable", String.valueOf(settings.isSmtpTls()));
props.put("mail.smtp.host", settings.getSmtpHost());
props.put("mail.smtp.port", String.valueOf(settings.getSmtpPort()));
props.put("mail.smtp.starttls.enable", String.valueOf(settings.get().tls()));
props.put("mail.smtp.host", settings.get().host());
props.put("mail.smtp.port", String.valueOf(settings.get().port()));
Session session = Session.getInstance(props, new Authenticator() {
@Override

View File

@@ -12,7 +12,6 @@ import javax.crypto.spec.PBEKeySpec;
import org.apache.commons.lang3.StringUtils;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -20,7 +19,7 @@ import lombok.extern.slf4j.Slf4j;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@SuppressWarnings("serial")
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class PasswordEncryptionService implements Serializable {

View File

@@ -24,11 +24,10 @@ import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class UserService {
@@ -117,8 +116,7 @@ public class UserService {
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(config.getApplicationSettings().getAllowRegistrations(),
"Registrations are closed on this CommaFeed instance");
Preconditions.checkState(config.users().allowRegistrations(), "Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");

View File

@@ -14,7 +14,6 @@ import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.Feed;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
@@ -35,7 +34,6 @@ public class DatabaseCleaningService {
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final Meter entriesDeletedMeter;
@Inject
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
@@ -43,7 +41,7 @@ public class DatabaseCleaningService {
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.getApplicationSettings().getDatabaseCleanupBatchSize();
this.batchSize = config.database().cleanup().batchSize();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}

View File

@@ -1,109 +1,41 @@
package com.commafeed.backend.service.db;
import java.util.HashMap;
import java.util.Map;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import org.kohsuke.MetaInfServices;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.service.UserService;
import io.dropwizard.lifecycle.Managed;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import liquibase.Scope;
import liquibase.UpdateSummaryEnum;
import liquibase.changelog.ChangeLogParameters;
import liquibase.command.CommandScope;
import liquibase.command.core.UpdateCommandStep;
import liquibase.command.core.helpers.DatabaseChangelogCommandStep;
import liquibase.command.core.helpers.DbUrlConnectionArgumentsCommandStep;
import liquibase.command.core.helpers.ShowSummaryArgument;
import liquibase.database.Database;
import liquibase.database.DatabaseFactory;
import liquibase.database.core.PostgresDatabase;
import liquibase.database.jvm.JdbcConnection;
import liquibase.exception.DatabaseException;
import liquibase.resource.ClassLoaderResourceAccessor;
import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class DatabaseStartupService implements Managed {
public class DatabaseStartupService {
private final UnitOfWork unitOfWork;
private final SessionFactory sessionFactory;
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
@Override
public void start() {
updateSchema();
public void populateInitialData() {
long count = unitOfWork.call(userDAO::count);
if (count == 0) {
unitOfWork.run(this::initialData);
}
}
private void updateSchema() {
log.info("checking if database schema needs updating");
try (Session session = sessionFactory.openSession()) {
session.doWork(connection -> {
try {
JdbcConnection jdbcConnection = new JdbcConnection(connection);
Database database = getDatabase(jdbcConnection);
Map<String, Object> scopeObjects = new HashMap<>();
scopeObjects.put(Scope.Attr.database.name(), database);
scopeObjects.put(Scope.Attr.resourceAccessor.name(),
new ClassLoaderResourceAccessor(Thread.currentThread().getContextClassLoader()));
Scope.child(scopeObjects, () -> {
CommandScope command = new CommandScope(UpdateCommandStep.COMMAND_NAME);
command.addArgumentValue(DbUrlConnectionArgumentsCommandStep.DATABASE_ARG, database);
command.addArgumentValue(UpdateCommandStep.CHANGELOG_FILE_ARG, "migrations.xml");
command.addArgumentValue(DatabaseChangelogCommandStep.CHANGELOG_PARAMETERS, new ChangeLogParameters(database));
command.addArgumentValue(ShowSummaryArgument.SHOW_SUMMARY, UpdateSummaryEnum.OFF);
command.execute();
});
} catch (Exception e) {
throw new RuntimeException(e);
}
});
}
log.info("database schema is up to date");
}
private Database getDatabase(JdbcConnection connection) throws DatabaseException {
Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(connection);
if (database instanceof PostgresDatabase) {
database = new PostgresDatabase() {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
};
database.setConnection(connection);
}
return database;
}
private void initialData() {
log.info("populating database with default values");
try {
userService.createAdminUser();
if (config.getApplicationSettings().getCreateDemoAccount()) {
if (config.users().createDemoAccount()) {
userService.createDemoUser();
}
} catch (Exception e) {
@@ -111,4 +43,20 @@ public class DatabaseStartupService implements Managed {
}
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
}

View File

@@ -1,87 +0,0 @@
package com.commafeed.backend.service.db;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import com.manticore.h2.H2MigrationTool;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
public class H2MigrationService {
private static final String H2_FILE_SUFFIX = ".mv.db";
public void migrateIfNeeded(Path path, String user, String password) {
if (Files.notExists(path)) {
return;
}
int format;
try {
format = getH2FileFormat(path);
} catch (IOException e) {
throw new RuntimeException("could not detect H2 format", e);
}
if (format == 2) {
try {
migrate(path, user, password, "2.1.214", "2.2.224");
} catch (Exception e) {
throw new RuntimeException("could not migrate H2 to format 3", e);
}
}
}
public int getH2FileFormat(Path path) throws IOException {
try (BufferedReader reader = Files.newBufferedReader(path)) {
String headers = reader.readLine();
return Stream.of(headers.split(","))
.filter(h -> h.startsWith("format:"))
.map(h -> h.split(":")[1])
.map(Integer::parseInt)
.findFirst()
.orElseThrow(() -> new RuntimeException("could not find format in H2 file headers"));
}
}
private void migrate(Path path, String user, String password, String fromVersion, String toVersion) throws Exception {
log.info("migrating H2 database at {} from format {} to format {}", path, fromVersion, toVersion);
Path scriptPath = path.resolveSibling("script-%d.sql".formatted(System.currentTimeMillis()));
Path newVersionPath = path.resolveSibling("%s.%s%s".formatted(StringUtils.removeEnd(path.getFileName().toString(), H2_FILE_SUFFIX),
getPatchVersion(toVersion), H2_FILE_SUFFIX));
Path oldVersionBackupPath = path.resolveSibling("%s.%s.backup".formatted(path.getFileName(), getPatchVersion(fromVersion)));
Files.deleteIfExists(scriptPath);
Files.deleteIfExists(newVersionPath);
Files.deleteIfExists(oldVersionBackupPath);
H2MigrationTool.readDriverRecords();
new H2MigrationTool().migrate(fromVersion, toVersion, path.toAbsolutePath().toString(), user, password,
scriptPath.toAbsolutePath().toString(), "", "", false, false, "");
if (!Files.exists(newVersionPath)) {
throw new RuntimeException("H2 migration failed, new version file not found");
}
Files.move(path, oldVersionBackupPath);
Files.move(newVersionPath, path);
Files.delete(oldVersionBackupPath);
Files.delete(scriptPath);
log.info("migrated H2 database from format {} to format {}", fromVersion, toVersion);
}
private String getPatchVersion(String version) {
return StringUtils.substringAfterLast(version, ".");
}
}

View File

@@ -7,11 +7,10 @@ import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class PostLoginActivities {

View File

@@ -9,12 +9,11 @@ import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
@Slf4j
public class DemoAccountCleanupTask extends ScheduledTask {
@@ -26,7 +25,7 @@ public class DemoAccountCleanupTask extends ScheduledTask {
@Override
protected void run() {
if (!config.getApplicationSettings().getCreateDemoAccount()) {
if (!config.users().createDemoAccount()) {
return;
}

View File

@@ -5,11 +5,10 @@ import java.util.concurrent.TimeUnit;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
@@ -18,7 +17,7 @@ public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
@Override
public void run() {
int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}

View File

@@ -7,11 +7,10 @@ import java.util.concurrent.TimeUnit;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class OldEntriesCleanupTask extends ScheduledTask {
@@ -20,9 +19,9 @@ public class OldEntriesCleanupTask extends ScheduledTask {
@Override
public void run() {
int maxAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays();
if (maxAgeDays > 0) {
Instant threshold = Instant.now().minus(Duration.ofDays(maxAgeDays));
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
cleaner.cleanEntriesOlderThan(threshold);
}
}

View File

@@ -6,11 +6,10 @@ import java.util.concurrent.TimeUnit;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class OldStatusesCleanupTask extends ScheduledTask {
@@ -19,7 +18,7 @@ public class OldStatusesCleanupTask extends ScheduledTask {
@Override
public void run() {
Instant threshold = config.getApplicationSettings().getUnreadThreshold();
Instant threshold = config.database().cleanup().statusesInstantThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}

View File

@@ -4,11 +4,10 @@ import java.util.concurrent.TimeUnit;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class OrphanedContentsCleanupTask extends ScheduledTask {

View File

@@ -4,11 +4,10 @@ import java.util.concurrent.TimeUnit;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class OrphanedFeedsCleanupTask extends ScheduledTask {

View File

@@ -0,0 +1,28 @@
package com.commafeed.backend.task;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import io.quarkus.arc.All;
import jakarta.inject.Singleton;
@Singleton
public class TaskScheduler {
private final List<ScheduledTask> tasks;
private final ScheduledExecutorService executor;
public TaskScheduler(@All List<ScheduledTask> tasks) {
this.tasks = tasks;
this.executor = Executors.newScheduledThreadPool(tasks.size());
}
public void start() {
tasks.forEach(task -> task.register(executor));
}
public void stop() {
executor.shutdownNow();
}
}

View File

@@ -4,6 +4,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import jakarta.inject.Singleton;
@Singleton
public class InPageReferenceFeedURLProvider implements FeedURLProvider {
@Override

View File

@@ -3,12 +3,15 @@ package com.commafeed.backend.urlprovider;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import jakarta.inject.Singleton;
/**
* Workaround for Youtube channels
*
* converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
* https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID
*/
@Singleton
public class YoutubeFeedURLProvider implements FeedURLProvider {
private static final Pattern REGEXP = Pattern.compile("(.*\\byoutube\\.com)\\/channel\\/([^\\/]+)", Pattern.CASE_INSENSITIVE);