mirror of
https://github.com/Athou/commafeed.git
synced 2026-11-20 10:43:37 +00:00
Use batched IN clause for GUID existence check during feed refresh
Instead of fetching all existing GUID hashes for a feed from the database only check the GUIDs present in the current feed XML using a SQL IN clause. GUIDs are partitioned into batches of 1000. Also adds an integration test verifying that refreshing an up-to-date feed does not create duplicate entries.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package com.commafeed.backend.dao;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
@@ -11,6 +12,7 @@ import jakarta.persistence.EntityManager;
|
||||
import com.commafeed.backend.model.Feed;
|
||||
import com.commafeed.backend.model.FeedEntry;
|
||||
import com.commafeed.backend.model.QFeedEntry;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.querydsl.core.Tuple;
|
||||
import com.querydsl.core.types.dsl.NumberExpression;
|
||||
import com.querydsl.jpa.impl.JPAQuery;
|
||||
@@ -19,6 +21,7 @@ import com.querydsl.jpa.impl.JPAQuery;
|
||||
public class FeedEntryDAO extends GenericDAO<FeedEntry> {
|
||||
|
||||
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
|
||||
private static final int IN_CLAUSE_BATCH_SIZE = 1000;
|
||||
|
||||
public FeedEntryDAO(EntityManager entityManager) {
|
||||
super(entityManager, FeedEntry.class);
|
||||
@@ -28,8 +31,16 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
|
||||
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
|
||||
}
|
||||
|
||||
public Set<String> findExistingGuids(Feed feed) {
|
||||
return new HashSet<>(query().select(ENTRY.guidHash).from(ENTRY).where(ENTRY.feed.eq(feed)).fetch());
|
||||
public Set<String> findExistingGuids(Feed feed, Set<String> guidHashes) {
|
||||
if (guidHashes.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
|
||||
Set<String> result = new HashSet<>();
|
||||
for (List<String> batch : Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) {
|
||||
result.addAll(query().select(ENTRY.guidHash).from(ENTRY).where(ENTRY.feed.eq(feed), ENTRY.guidHash.in(batch)).fetch());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max, boolean keepStarredEntries) {
|
||||
|
||||
Reference in New Issue
Block a user