batch guid existence check on feed refresh

Instead of querying the database once per entry to check if it already
exists, fetch all existing guids for a feed in a single query upfront
and filter new entries in memory before the per-entry loop.

The per-entry findExisting call inside addEntry is kept as a safety net
for concurrent inserts, but is now only reached for new entries.

Profiling in real usage showed a ~50% reduction in guidHash queries,
with the check falling from the most expensive query to sixth in the ranking.

commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java

@@ -1,7 +1,9 @@
 package com.commafeed.backend.dao;
 
 import java.time.Instant;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import jakarta.inject.Singleton;
 import jakarta.persistence.EntityManager;
@@ -26,6 +28,10 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
 		return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
 	}
 
+	public Set<String> findExistingGuids(Feed feed) {
+		return new HashSet<>(query().select(ENTRY.guidHash).from(ENTRY).where(ENTRY.feed.eq(feed)).fetch());
+	}
+
 	public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max, boolean keepStarredEntries) {
 		NumberExpression<Long> count = ENTRY.id.count();
 		JPAQuery<Tuple> query = query().select(ENTRY.feed.id, count).from(ENTRY);

commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java

@@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.MetricRegistry;
 import com.commafeed.backend.Digests;
+import com.commafeed.backend.dao.FeedEntryDAO;
 import com.commafeed.backend.dao.FeedSubscriptionDAO;
 import com.commafeed.backend.dao.UnitOfWork;
 import com.commafeed.backend.feed.parser.FeedParserResult.Content;
@@ -42,6 +43,7 @@ public class FeedRefreshUpdater {
 	private final UnitOfWork unitOfWork;
 	private final FeedService feedService;
 	private final FeedEntryService feedEntryService;
+	private final FeedEntryDAO feedEntryDAO;
 	private final FeedSubscriptionDAO feedSubscriptionDAO;
 
 	private final Striped<Lock> locks;
@@ -49,11 +51,12 @@ public class FeedRefreshUpdater {
 	private final Meter feedUpdated;
 	private final Meter entryInserted;
 
-	public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
-			FeedSubscriptionDAO feedSubscriptionDAO) {
+	public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, FeedEntryDAO feedEntryDAO,
+			MetricRegistry metrics, FeedSubscriptionDAO feedSubscriptionDAO) {
 		this.unitOfWork = unitOfWork;
 		this.feedService = feedService;
 		this.feedEntryService = feedEntryService;
+		this.feedEntryDAO = feedEntryDAO;
 		this.feedSubscriptionDAO = feedSubscriptionDAO;
 
 		locks = Striped.lazyWeakLock(100000);
@@ -126,8 +129,11 @@ public class FeedRefreshUpdater {
 		Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription = new HashMap<>();
 
 		if (!entries.isEmpty()) {
+			Set<String> existingGuids = unitOfWork.call(() -> feedEntryDAO.findExistingGuids(feed));
+			List<Entry> newEntries = entries.stream().filter(e -> !existingGuids.contains(Digests.sha1Hex(e.guid()))).toList();
+
 			List<FeedSubscription> subscriptions = null;
-			for (Entry entry : entries) {
+			for (Entry entry : newEntries) {
 				if (subscriptions == null) {
 					subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
 				}

commafeed-server/src/test/java/com/commafeed/integration/rest/LargeDatasetIT.java

@@ -0,0 +1,117 @@
+package com.commafeed.integration.rest;
+
+import java.time.Duration;
+import java.time.Instant;
+
+import org.awaitility.Awaitility;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockserver.model.HttpRequest;
+import org.mockserver.model.HttpResponse;
+import org.mockserver.model.MediaType;
+
+import com.commafeed.TestConstants;
+import com.commafeed.frontend.model.Entries;
+import com.commafeed.frontend.model.Entry;
+import com.commafeed.frontend.model.request.MarkRequest;
+import com.commafeed.frontend.resource.CategoryREST;
+import com.commafeed.integration.BaseIT;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.RestAssured;
+import io.restassured.http.ContentType;
+
+@QuarkusTest
+class LargeDatasetIT extends BaseIT {
+
+	private static final int FEED_COUNT = 10;
+	private static final int ENTRIES_PER_FEED = 20;
+	private static final int TOTAL_ENTRIES = FEED_COUNT * ENTRIES_PER_FEED;
+
+	@BeforeEach
+	void setup() {
+		initialSetup(TestConstants.ADMIN_USERNAME, TestConstants.ADMIN_PASSWORD);
+		RestAssured.authentication = RestAssured.preemptive().basic(TestConstants.ADMIN_USERNAME, TestConstants.ADMIN_PASSWORD);
+
+		for (int i = 0; i < FEED_COUNT; i++) {
+			String path = "/feed/" + i;
+			getMockServerClient().when(HttpRequest.request().withMethod("GET").withPath(path))
+					.respond(HttpResponse.response().withBody(generateFeed(i)).withContentType(MediaType.APPLICATION_XML));
+			subscribe("http://localhost:" + getMockServerClient().getPort() + path);
+		}
+
+		Awaitility.await().atMost(Duration.ofSeconds(60)).until(() -> getAllEntries().getEntries().size(), count -> count >= TOTAL_ENTRIES);
+	}
+
+	@AfterEach
+	void cleanup() {
+		RestAssured.reset();
+	}
+
+	@Test
+	void markAllAsRead() {
+		Entries before = getAllEntries();
+		Assertions.assertEquals(TOTAL_ENTRIES, before.getEntries().size());
+		Assertions.assertTrue(before.getEntries().stream().noneMatch(Entry::isRead));
+
+		MarkRequest markRequest = new MarkRequest();
+		markRequest.setId(CategoryREST.ALL);
+		markRequest.setRead(true);
+		RestAssured.given().body(markRequest).contentType(ContentType.JSON).post("rest/category/mark").then().statusCode(200);
+
+		Entries after = getAllEntries();
+		Assertions.assertEquals(TOTAL_ENTRIES, after.getEntries().size());
+		Assertions.assertTrue(after.getEntries().stream().allMatch(Entry::isRead));
+	}
+
+	@Test
+	void paginationHasMore() {
+		Entries firstPage = RestAssured.given()
+				.get("rest/category/entries?id=all&readType=all&limit=20&offset=0")
+				.then()
+				.statusCode(200)
+				.extract()
+				.as(Entries.class);
+		Assertions.assertEquals(20, firstPage.getEntries().size());
+		Assertions.assertTrue(firstPage.isHasMore());
+
+		Entries lastPage = RestAssured.given()
+				.get("rest/category/entries?id=all&readType=all&limit=20&offset={offset}", TOTAL_ENTRIES - 20)
+				.then()
+				.statusCode(200)
+				.extract()
+				.as(Entries.class);
+		Assertions.assertEquals(20, lastPage.getEntries().size());
+		Assertions.assertFalse(lastPage.isHasMore());
+	}
+
+	private Entries getAllEntries() {
+		return RestAssured.given()
+				.get("rest/category/entries?id=all&readType=all&limit=1000")
+				.then()
+				.statusCode(200)
+				.extract()
+				.as(Entries.class);
+	}
+
+	private String generateFeed(int feedIndex) {
+		StringBuilder sb = new StringBuilder();
+		sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<rss version=\"2.0\">\n<channel>\n");
+		sb.append("<title>Feed ").append(feedIndex).append("</title>\n");
+		sb.append("<link>https://hostname.local/feed/").append(feedIndex).append("</link>\n");
+		sb.append("<description>Test feed ").append(feedIndex).append("</description>\n");
+		Instant base = Instant.parse("2024-01-01T00:00:00Z");
+		for (int i = 0; i < ENTRIES_PER_FEED; i++) {
+			sb.append("<item>\n");
+			sb.append("<title>Feed ").append(feedIndex).append(" Item ").append(i).append("</title>\n");
+			sb.append("<link>https://hostname.local/feed/").append(feedIndex).append("/item/").append(i).append("</link>\n");
+			sb.append("<description>Description for feed ").append(feedIndex).append(" item ").append(i).append("</description>\n");
+			sb.append("<pubDate>").append(base.minus(Duration.ofHours(i))).append("</pubDate>\n");
+			sb.append("</item>\n");
+		}
+		sb.append("</channel>\n</rss>");
+		return sb.toString();
+	}
+}