Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-22 12:04:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use batched IN clause for GUID existence check during feed refresh

Browse Source 
Instead of fetching all existing GUID hashes for a feed from the database only check the GUIDs present in the current feed XML using a SQL IN clause. GUIDs are partitioned into batches of 1000.

Also adds an integration test verifying that refreshing an up-to-date feed does not create duplicate entries.
This commit is contained in:
Ingo Kegel
2026-04-10 10:13:37 +02:00
parent 16036897cb
commit 1d23907652
3 changed files with 41 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
View File

@@ -1,6 +1,7 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -11,6 +12,7 @@ import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry;
import com.google.common.collect.Lists;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression;
import com.querydsl.jpa.impl.JPAQuery;
@@ -19,6 +21,7 @@ import com.querydsl.jpa.impl.JPAQuery;
public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final int IN_CLAUSE_BATCH_SIZE = 1000;
public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class);
@@ -28,8 +31,16 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
}
public Set<String> findExistingGuids(Feed feed) {
return new HashSet<>(query().select(ENTRY.guidHash).from(ENTRY).where(ENTRY.feed.eq(feed)).fetch());
public Set<String> findExistingGuids(Feed feed, Set<String> guidHashes) {
if (guidHashes.isEmpty()) {
return Set.of();
}
Set<String> result = new HashSet<>();
for (List<String> batch : Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) {
result.addAll(query().select(ENTRY.guidHash).from(ENTRY).where(ENTRY.feed.eq(feed), ENTRY.guidHash.in(batch)).fetch());
}
return result;
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max, boolean keepStarredEntries) {

12
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -129,8 +129,16 @@ public class FeedRefreshUpdater {
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
Set<String> existingGuids = unitOfWork.call(() -> feedEntryDAO.findExistingGuids(feed));
List<Entry> newEntries = entries.stream().filter(e -> !existingGuids.contains(Digests.sha1Hex(e.guid()))).toList();
Map<String, Entry> entriesByGuidHash = new HashMap<>();
for (Entry entry : entries) {
entriesByGuidHash.put(Digests.sha1Hex(entry.guid()), entry);
}
Set<String> existingGuids = unitOfWork.call(() -> feedEntryDAO.findExistingGuids(feed, entriesByGuidHash.keySet()));
List<Entry> newEntries = entriesByGuidHash.entrySet()
.stream()
.filter(e -> !existingGuids.contains(e.getKey()))
.map(Map.Entry::getValue)
.toList();
List<FeedSubscription> subscriptions = null;
for (Entry entry : newEntries) {

19
commafeed-server/src/test/java/com/commafeed/integration/rest/LargeDatasetIT.java
View File

@@ -30,6 +30,8 @@ class LargeDatasetIT extends BaseIT {
private static final int ENTRIES_PER_FEED = 20;
private static final int TOTAL_ENTRIES = FEED_COUNT * ENTRIES_PER_FEED;
private Long firstSubscriptionId;
@BeforeEach
void setup() {
initialSetup(TestConstants.ADMIN_USERNAME, TestConstants.ADMIN_PASSWORD);
@@ -39,7 +41,10 @@ class LargeDatasetIT extends BaseIT {
String path = "/feed/" + i;
getMockServerClient().when(HttpRequest.request().withMethod("GET").withPath(path))
.respond(HttpResponse.response().withBody(generateFeed(i)).withContentType(MediaType.APPLICATION_XML));
subscribe("http://localhost:" + getMockServerClient().getPort() + path);
Long subscriptionId = subscribe("http://localhost:" + getMockServerClient().getPort() + path);
if (i == 0) {
firstSubscriptionId = subscriptionId;
}
}
Awaitility.await().atMost(Duration.ofSeconds(60)).until(() -> getAllEntries().getEntries().size(), count -> count >= TOTAL_ENTRIES);
@@ -66,6 +71,18 @@ class LargeDatasetIT extends BaseIT {
Assertions.assertTrue(after.getEntries().stream().allMatch(Entry::isRead));
}
@Test
void refreshDoesNotCreateDuplicateEntries() {
Assertions.assertEquals(TOTAL_ENTRIES, getAllEntries().getEntries().size());
Instant threshold = Instant.now().minus(Duration.ofSeconds(1));
forceRefreshAllFeeds();
Awaitility.await()
.atMost(Duration.ofSeconds(15))
.until(() -> getSubscription(firstSubscriptionId), f -> f.getLastRefresh().isAfter(threshold));
Assertions.assertEquals(TOTAL_ENTRIES, getAllEntries().getEntries().size());
}
@Test
void paginationHasMore() {
Entries firstPage = RestAssured.given()