Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-22 20:14:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'perf/batch-guid-check-on-feed-refresh' of https://github.com/ingokegel/commafeed into ingokegel-perf/batch-guid-check-on-feed-refresh

Browse Source
This commit is contained in:
Athou
2026-04-10 14:22:32 +02:00
parent efc6c10ace 1d23907652
commit ec35d846cf
3 changed files with 168 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
View File

@@ -1,7 +1,10 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@@ -9,6 +12,7 @@ import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry;
import com.google.common.collect.Lists;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression;
import com.querydsl.jpa.impl.JPAQuery;
@@ -17,6 +21,7 @@ import com.querydsl.jpa.impl.JPAQuery;
public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final int IN_CLAUSE_BATCH_SIZE = 1000;
public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class);
@@ -26,6 +31,18 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
}
public Set<String> findExistingGuids(Feed feed, Set<String> guidHashes) {
if (guidHashes.isEmpty()) {
return Set.of();
}
Set<String> result = new HashSet<>();
for (List<String> batch : Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) {
result.addAll(query().select(ENTRY.guidHash).from(ENTRY).where(ENTRY.feed.eq(feed), ENTRY.guidHash.in(batch)).fetch());
}
return result;
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max, boolean keepStarredEntries) {
NumberExpression<Long> count = ENTRY.id.count();
JPAQuery<Tuple> query = query().select(ENTRY.feed.id, count).from(ENTRY);

20
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
@@ -42,6 +43,7 @@ public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedEntryDAO feedEntryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final Striped<Lock> locks;
@@ -49,11 +51,12 @@ public class FeedRefreshUpdater {
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO) {
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, FeedEntryDAO feedEntryDAO,
MetricRegistry metrics, FeedSubscriptionDAO feedSubscriptionDAO) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedEntryDAO = feedEntryDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
locks = Striped.lazyWeakLock(100000);
@@ -126,8 +129,19 @@ public class FeedRefreshUpdater {
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
Map<String, Entry> entriesByGuidHash = new HashMap<>();
for (Entry entry : entries) {
entriesByGuidHash.put(Digests.sha1Hex(entry.guid()), entry);
}
Set<String> existingGuids = unitOfWork.call(() -> feedEntryDAO.findExistingGuids(feed, entriesByGuidHash.keySet()));
List<Entry> newEntries = entriesByGuidHash.entrySet()
.stream()
.filter(e -> !existingGuids.contains(e.getKey()))
.map(Map.Entry::getValue)
.toList();
List<FeedSubscription> subscriptions = null;
for (Entry entry : newEntries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}