Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-23 12:35:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

batch guid existence check on feed refresh

Browse Source 
Instead of querying the database once per entry to check if it already
exists, fetch all existing guids for a feed in a single query upfront
and filter new entries in memory before the per-entry loop.

The per-entry findExisting call inside addEntry is kept as a safety net
for concurrent inserts, but is now only reached for new entries.

Profiling in real usage showed a ~50% reduction in guidHash queries,
with the check falling from the most expensive query to sixth in the ranking.
This commit is contained in:
Ingo Kegel
2026-03-18 18:48:36 +01:00
parent ecbf8bec23
commit 76dbbdfbf9
3 changed files with 132 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
View File

@@ -1,7 +1,9 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@@ -26,6 +28,10 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
}
public Set<String> findExistingGuids(Feed feed) {
return new HashSet<>(query().select(ENTRY.guidHash).from(ENTRY).where(ENTRY.feed.eq(feed)).fetch());
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max, boolean keepStarredEntries) {
NumberExpression<Long> count = ENTRY.id.count();
JPAQuery<Tuple> query = query().select(ENTRY.feed.id, count).from(ENTRY);

12
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
@@ -42,6 +43,7 @@ public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedEntryDAO feedEntryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final Striped<Lock> locks;
@@ -49,11 +51,12 @@ public class FeedRefreshUpdater {
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO) {
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, FeedEntryDAO feedEntryDAO,
MetricRegistry metrics, FeedSubscriptionDAO feedSubscriptionDAO) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedEntryDAO = feedEntryDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
locks = Striped.lazyWeakLock(100000);
@@ -126,8 +129,11 @@ public class FeedRefreshUpdater {
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
Set<String> existingGuids = unitOfWork.call(() -> feedEntryDAO.findExistingGuids(feed));
List<Entry> newEntries = entries.stream().filter(e -> !existingGuids.contains(Digests.sha1Hex(e.guid()))).toList();
List<FeedSubscription> subscriptions = null;
for (Entry entry : entries) {
for (Entry entry : newEntries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}