Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-24 04:57:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor

Browse Source
This commit is contained in:
Athou
2026-02-18 16:03:43 +01:00
parent 77bb948bf2
commit 2be61e8b1c
73 changed files with 1691 additions and 1556 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -13,7 +13,6 @@ import org.apache.commons.lang3.Strings;
import com.commafeed.backend.Digests;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
import com.commafeed.backend.HttpGetter.HttpRequest;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
@@ -46,7 +45,7 @@ public class FeedFetcher {
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedParsingException, IOException, NotModifiedException,
TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException, NoFeedFoundException {
TooManyRequestsException, SchemeNotAllowedException, NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());

40
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java
View File

@@ -7,6 +7,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -21,6 +22,8 @@ import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import lombok.extern.slf4j.Slf4j;
@@ -32,6 +35,7 @@ public class FeedRefreshEngine {
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final FeedUpdateNotifier notifier;
private final CommaFeedConfiguration config;
private final Meter refill;
@@ -42,13 +46,15 @@ public class FeedRefreshEngine {
private final ExecutorService refillExecutor;
private final ThreadPoolExecutor workerExecutor;
private final ThreadPoolExecutor databaseUpdaterExecutor;
private final ThreadPoolExecutor notifierExecutor;
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
CommaFeedConfiguration config, MetricRegistry metrics) {
FeedUpdateNotifier notifier, CommaFeedConfiguration config, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.notifier = notifier;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
@@ -59,10 +65,14 @@ public class FeedRefreshEngine {
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
this.notifierExecutor = newDiscardingExecutorService(config.pushNotifications().threads(),
config.pushNotifications().queueCapacity());
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) workerExecutor::getActiveCount);
metrics.register(MetricRegistry.name(getClass(), "updater", "active"), (Gauge<Integer>) databaseUpdaterExecutor::getActiveCount);
metrics.register(MetricRegistry.name(getClass(), "notifier", "active"), (Gauge<Integer>) notifierExecutor::getActiveCount);
metrics.register(MetricRegistry.name(getClass(), "notifier", "queue"), (Gauge<Integer>) () -> notifierExecutor.getQueue().size());
}
public void start() {
@@ -152,10 +162,19 @@ public class FeedRefreshEngine {
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while processing feed {}", feed.getUrl(), ex);
}
.thenCompose(r -> {
List<CompletableFuture<Void>> futures = r.insertedUnreadEntriesBySubscription().entrySet().stream().map(e -> {
FeedSubscription sub = e.getKey();
List<FeedEntry> entries = e.getValue();
notifier.notifyOverWebsocket(sub, entries);
return CompletableFuture.runAsync(() -> notifier.sendPushNotifications(sub, entries), notifierExecutor);
}).toList();
return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new));
})
.exceptionally(ex -> {
log.error("error while processing feed {}", feed.getUrl(), ex);
return null;
});
}
@@ -183,6 +202,7 @@ public class FeedRefreshEngine {
this.refillExecutor.shutdownNow();
this.workerExecutor.shutdownNow();
this.databaseUpdaterExecutor.shutdownNow();
this.notifierExecutor.shutdownNow();
}
/**
@@ -194,6 +214,16 @@ public class FeedRefreshEngine {
return pool;
}
/**
* returns an ExecutorService that discards tasks if the queue is full
*/
private ThreadPoolExecutor newDiscardingExecutorService(int threads, int queueCapacity) {
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(queueCapacity));
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService that blocks submissions until a thread is available
*/

78
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -20,19 +20,14 @@ import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.UserSettings;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.backend.service.NotificationService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import com.google.common.util.concurrent.Striped;
import lombok.extern.slf4j.Slf4j;
@@ -48,9 +43,6 @@ public class FeedRefreshUpdater {
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserSettingsDAO userSettingsDAO;
private final WebSocketSessions webSocketSessions;
private final NotificationService notificationService;
private final Striped<Lock> locks;
@@ -58,15 +50,11 @@ public class FeedRefreshUpdater {
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO, UserSettingsDAO userSettingsDAO, WebSocketSessions webSocketSessions,
NotificationService notificationService) {
FeedSubscriptionDAO feedSubscriptionDAO) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.userSettingsDAO = userSettingsDAO;
this.webSocketSessions = webSocketSessions;
this.notificationService = notificationService;
locks = Striped.lazyWeakLock(100000);
@@ -100,19 +88,20 @@ public class FeedRefreshUpdater {
if (locked1 && locked2) {
processed = true;
insertedEntry = unitOfWork.call(() -> {
FeedEntry feedEntry = feedEntryService.find(feed, entry);
if (feedEntry == null) {
feedEntry = feedEntryService.create(feed, entry);
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread = feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
return feedEntry;
if (feedEntryService.find(feed, entry) != null) {
// entry already exists, nothing to do
return null;
}
return null;
FeedEntry feedEntry = feedEntryService.create(feed, entry);
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread = feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
return feedEntry;
});
} else {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
@@ -131,11 +120,10 @@ public class FeedRefreshUpdater {
return new AddEntryResult(processed, insertedEntry, subscriptionsForWhichEntryIsUnread);
}
public boolean update(Feed feed, List<Entry> entries) {
public FeedRefreshUpdaterResult update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, Long> unreadCountBySubscription = new HashMap<>();
Map<FeedSubscription, List<FeedEntry>> insertedEntriesBySubscription = new HashMap<>();
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
@@ -147,9 +135,8 @@ public class FeedRefreshUpdater {
processed &= addEntryResult.processed;
inserted += addEntryResult.insertedEntry != null ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> {
unreadCountBySubscription.merge(sub, 1L, Long::sum);
if (addEntryResult.insertedEntry != null) {
insertedEntriesBySubscription.computeIfAbsent(sub, k -> new ArrayList<>()).add(addEntryResult.insertedEntry);
insertedUnreadEntriesBySubscription.computeIfAbsent(sub, k -> new ArrayList<>()).add(addEntryResult.insertedEntry);
}
});
}
@@ -172,36 +159,13 @@ public class FeedRefreshUpdater {
unitOfWork.run(() -> feedService.update(feed));
notifyOverWebsocket(unreadCountBySubscription);
sendNotifications(insertedEntriesBySubscription);
return processed;
}
private void notifyOverWebsocket(Map<FeedSubscription, Long> unreadCountBySubscription) {
unreadCountBySubscription.forEach((sub, unreadCount) -> webSocketSessions.sendMessage(sub.getUser(),
WebSocketMessageBuilder.newFeedEntries(sub, unreadCount)));
}
private void sendNotifications(Map<FeedSubscription, List<FeedEntry>> insertedEntriesBySubscription) {
insertedEntriesBySubscription.forEach((sub, feedEntries) -> {
if (!sub.isNotifyOnNewEntries()) {
return;
}
try {
UserSettings settings = unitOfWork.call(() -> userSettingsDAO.findByUser(sub.getUser()));
if (settings != null && settings.isNotificationEnabled()) {
for (FeedEntry feedEntry : feedEntries) {
notificationService.notify(settings, sub, feedEntry);
}
}
} catch (Exception e) {
log.error("error sending push notification for subscription {}", sub.getId(), e);
}
});
return new FeedRefreshUpdaterResult(insertedUnreadEntriesBySubscription);
}
private record AddEntryResult(boolean processed, FeedEntry insertedEntry, Set<FeedSubscription> subscriptionsForWhichEntryIsUnread) {
}
public record FeedRefreshUpdaterResult(Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription) {
}
}

50
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUpdateNotifier.java Normal file
View File

@@ -0,0 +1,50 @@
package com.commafeed.backend.feed;
import java.util.List;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.UserSettings;
import com.commafeed.backend.service.PushNotificationService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
@RequiredArgsConstructor
public class FeedUpdateNotifier {
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserSettingsDAO userSettingsDAO;
private final WebSocketSessions webSocketSessions;
private final PushNotificationService pushNotificationService;
public void notifyOverWebsocket(FeedSubscription sub, List<FeedEntry> entries) {
if (!entries.isEmpty()) {
webSocketSessions.sendMessage(sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub, entries.size()));
}
}
public void sendPushNotifications(FeedSubscription sub, List<FeedEntry> entries) {
if (!config.pushNotifications().enabled() || !sub.isPushNotificationsEnabled() || entries.isEmpty()) {
return;
}
UserSettings settings = unitOfWork.call(() -> userSettingsDAO.findByUser(sub.getUser()));
if (settings != null && settings.getPushNotificationType() != null) {
for (FeedEntry entry : entries) {
pushNotificationService.notify(settings, sub, entry);
}
}
}
}