Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-25 21:45:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Athou
2024-08-07 08:10:14 +02:00
parent 2f6ddf0e70
commit cc32f8ad16
164 changed files with 2011 additions and 3288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -2,7 +2,7 @@ package com.commafeed.backend.feed;
import java.io.IOException;
import java.time.Instant;
import java.util.Set;
import java.util.List;
import org.apache.commons.codec.binary.StringUtils;
@@ -15,22 +15,26 @@ import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import com.rometools.rome.io.FeedException;
import jakarta.inject.Inject;
import io.quarkus.arc.All;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches a feed then parses it
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final Set<FeedURLProvider> urlProviders;
private final List<FeedURLProvider> urlProviders;
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedException, IOException, NotModifiedException {
@@ -87,7 +91,7 @@ public class FeedFetcher {
result.getDuration());
}
private static String extractFeedUrl(Set<FeedURLProvider> urlProviders, String url, String urlContent) {
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
for (FeedURLProvider urlProvider : urlProviders) {
String feedUrl = urlProvider.get(url, urlContent);
if (feedUrl != null) {

21
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java
View File

@@ -1,6 +1,5 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
@@ -21,14 +20,12 @@ import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import io.dropwizard.lifecycle.Managed;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshEngine implements Managed {
public class FeedRefreshEngine {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
@@ -45,7 +42,6 @@ public class FeedRefreshEngine implements Managed {
private final ThreadPoolExecutor workerExecutor;
private final ThreadPoolExecutor databaseUpdaterExecutor;
@Inject
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
CommaFeedConfiguration config, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
@@ -60,15 +56,14 @@ public class FeedRefreshEngine implements Managed {
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.getApplicationSettings().getBackgroundThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.getApplicationSettings().getDatabaseUpdateThreads());
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) workerExecutor::getActiveCount);
metrics.register(MetricRegistry.name(getClass(), "updater", "active"), (Gauge<Integer>) databaseUpdaterExecutor::getActiveCount);
}
@Override
public void start() {
startFeedProcessingLoop();
startRefillLoop();
@@ -165,22 +160,20 @@ public class FeedRefreshEngine implements Managed {
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> {
Instant lastLoginThreshold = Boolean.TRUE.equals(config.getApplicationSettings().getHeavyLoad())
? Instant.now().minus(Duration.ofDays(30))
: null;
Instant lastLoginThreshold = config.feedRefresh().userInactivityPeriod().isZero() ? null
: Instant.now().minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Instant nextUpdateDate = Instant.now().plus(Duration.ofMinutes(config.getApplicationSettings().getRefreshIntervalMinutes()));
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
return feeds;
});
}
private int getBatchSize() {
return Math.min(100, 3 * config.getApplicationSettings().getBackgroundThreads());
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
@Override
public void stop() {
this.feedProcessingLoopExecutor.shutdownNow();
this.refillLoopExecutor.shutdownNow();

16
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshIntervalCalculator.java
View File

@@ -6,24 +6,22 @@ import java.time.temporal.ChronoUnit;
import com.commafeed.CommaFeedConfiguration;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
@Singleton
public class FeedRefreshIntervalCalculator {
private final boolean heavyLoad;
private final int refreshIntervalMinutes;
private final Duration refreshInterval;
private final boolean empiricalInterval;
@Inject
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config) {
this.heavyLoad = config.getApplicationSettings().getHeavyLoad();
this.refreshIntervalMinutes = config.getApplicationSettings().getRefreshIntervalMinutes();
this.refreshInterval = config.feedRefresh().interval();
this.empiricalInterval = config.feedRefresh().intervalEmpirical();
}
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval) {
Instant defaultRefreshInterval = getDefaultRefreshInterval();
return heavyLoad ? computeRefreshIntervalForHeavyLoad(publishedDate, averageEntryInterval, defaultRefreshInterval)
return empiricalInterval ? computeRefreshIntervalForHeavyLoad(publishedDate, averageEntryInterval, defaultRefreshInterval)
: defaultRefreshInterval;
}
@@ -33,7 +31,7 @@ public class FeedRefreshIntervalCalculator {
public Instant onFetchError(int errorCount) {
int retriesBeforeDisable = 3;
if (errorCount < retriesBeforeDisable || !heavyLoad) {
if (errorCount < retriesBeforeDisable || !empiricalInterval) {
return getDefaultRefreshInterval();
}
@@ -42,7 +40,7 @@ public class FeedRefreshIntervalCalculator {
}
private Instant getDefaultRefreshInterval() {
return Instant.now().plus(Duration.ofMinutes(refreshIntervalMinutes));
return Instant.now().plus(refreshInterval);
}
private Instant computeRefreshIntervalForHeavyLoad(Instant publishedDate, Long averageEntryInterval, Instant defaultRefreshInterval) {

2
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -32,7 +32,6 @@ import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import com.google.common.util.concurrent.Striped;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -58,7 +57,6 @@ public class FeedRefreshUpdater {
private final Meter feedUpdated;
private final Meter entryInserted;
@Inject
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO, CacheService cache, WebSocketSessions webSocketSessions) {
this.unitOfWork = unitOfWork;

10
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -16,7 +16,6 @@ import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
@@ -32,7 +31,6 @@ public class FeedRefreshWorker {
private final CommaFeedConfiguration config;
private final Meter feedFetched;
@Inject
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
@@ -51,14 +49,14 @@ public class FeedRefreshWorker {
List<Entry> entries = result.feed().entries();
Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
Integer maxEntriesAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays();
if (maxEntriesAgeDays > 0) {
Instant threshold = Instant.now().minus(Duration.ofDays(maxEntriesAgeDays));
Duration maxEntriesAgeDays = config.database().cleanup().entriesMaxAge();
if (!maxEntriesAgeDays.isZero()) {
Instant threshold = Instant.now().minus(maxEntriesAgeDays);
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList();
}

3
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
View File

@@ -38,14 +38,13 @@ import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedInput;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
/**
* Parses raw xml into a FeedParserResult object
*/
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@RequiredArgsConstructor
@Singleton
public class FeedParser {