refactor

commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java

@@ -31,7 +31,7 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
 		return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
 	}
 
-	public Set<String> findExistingGuids(Feed feed, Set<String> guidHashes) {
+	public Set<String> findExistingGuidHashes(Set<String> guidHashes, Feed feed) {
 		if (guidHashes.isEmpty()) {
 			return Set.of();
 		}

commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java

@@ -18,7 +18,6 @@ import org.apache.commons.lang3.StringUtils;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.MetricRegistry;
 import com.commafeed.backend.Digests;
-import com.commafeed.backend.dao.FeedEntryDAO;
 import com.commafeed.backend.dao.FeedSubscriptionDAO;
 import com.commafeed.backend.dao.UnitOfWork;
 import com.commafeed.backend.feed.parser.FeedParserResult.Content;
@@ -43,7 +42,6 @@ public class FeedRefreshUpdater {
 	private final UnitOfWork unitOfWork;
 	private final FeedService feedService;
 	private final FeedEntryService feedEntryService;
-	private final FeedEntryDAO feedEntryDAO;
 	private final FeedSubscriptionDAO feedSubscriptionDAO;
 
 	private final Striped<Lock> locks;
@@ -51,12 +49,11 @@ public class FeedRefreshUpdater {
 	private final Meter feedUpdated;
 	private final Meter entryInserted;
 
-	public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, FeedEntryDAO feedEntryDAO,
+	public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
-			MetricRegistry metrics, FeedSubscriptionDAO feedSubscriptionDAO) {
+			FeedSubscriptionDAO feedSubscriptionDAO) {
 		this.unitOfWork = unitOfWork;
 		this.feedService = feedService;
 		this.feedEntryService = feedEntryService;
-		this.feedEntryDAO = feedEntryDAO;
 		this.feedSubscriptionDAO = feedSubscriptionDAO;
 
 		locks = Striped.lazyWeakLock(100000);
@@ -129,18 +126,8 @@ public class FeedRefreshUpdater {
 		Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription = new HashMap<>();
 
 		if (!entries.isEmpty()) {
-			Map<String, Entry> entriesByGuidHash = new HashMap<>();
-			for (Entry entry : entries) {
-				entriesByGuidHash.put(Digests.sha1Hex(entry.guid()), entry);
-			}
-			Set<String> existingGuids = unitOfWork.call(() -> feedEntryDAO.findExistingGuids(feed, entriesByGuidHash.keySet()));
-			List<Entry> newEntries = entriesByGuidHash.entrySet()
-					.stream()
-					.filter(e -> !existingGuids.contains(e.getKey()))
-					.map(Map.Entry::getValue)
-					.toList();
-
 			List<FeedSubscription> subscriptions = null;
+			List<Entry> newEntries = unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries));
 			for (Entry entry : newEntries) {
 				if (subscriptions == null) {
 					subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));

commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java

@@ -2,6 +2,8 @@ package com.commafeed.backend.service;
 
 import java.time.Instant;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import jakarta.inject.Singleton;
 
@@ -52,6 +54,12 @@ public class FeedEntryService {
 		return feedEntry;
 	}
 
+	public List<Entry> removeExistingEntries(Feed feed, List<Entry> entries) {
+		Set<String> guidHashes = entries.stream().map(e -> Digests.sha1Hex(e.guid())).collect(Collectors.toSet());
+		Set<String> existingGuidHashes = feedEntryDAO.findExistingGuidHashes(guidHashes, feed);
+		return entries.stream().filter(e -> !existingGuidHashes.contains(Digests.sha1Hex(e.guid()))).toList();
+	}
+
 	public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
 		boolean matches = true;
 		try {

commafeed-server/src/test/java/com/commafeed/integration/BaseIT.java

@@ -174,6 +174,15 @@ public abstract class BaseIT {
 				.as(Entries.class);
 	}
 
+	protected Entries getCategoryEntries(String categoryId, int offset, int limit) {
+		return RestAssured.given()
+				.get("rest/category/entries?id={id}&readType=all&offset={offset}&limit={limit}", categoryId, offset, limit)
+				.then()
+				.statusCode(HttpStatus.SC_OK)
+				.extract()
+				.as(Entries.class);
+	}
+
 	protected Entries getCategoryEntries(String categoryId, String keywords) {
 		return RestAssured.given()
 				.get("rest/category/entries?id={id}&readType=all&keywords={keywords}", categoryId, keywords)

commafeed-server/src/test/java/com/commafeed/integration/rest/CategoryIT.java

@@ -159,6 +159,19 @@ class CategoryIT extends BaseIT {
 			Assertions.assertEquals(2, entries.getEntries().size());
 		}
 
+		@Test
+		void pagination() {
+			subscribeAndWaitForEntries(getFeedUrl());
+
+			Entries firstPage = getCategoryEntries(CategoryREST.ALL, 0, 1);
+			Assertions.assertEquals(1, firstPage.getEntries().size());
+			Assertions.assertTrue(firstPage.isHasMore());
+
+			Entries lastPage = getCategoryEntries(CategoryREST.ALL, 1, 10);
+			Assertions.assertEquals(1, lastPage.getEntries().size());
+			Assertions.assertFalse(lastPage.isHasMore());
+		}
+
 		@Test
 		void allAsFeed() throws FeedException {
 			subscribeAndWaitForEntries(getFeedUrl());

commafeed-server/src/test/java/com/commafeed/integration/rest/FeedIT.java

@@ -31,6 +31,7 @@ import com.commafeed.frontend.model.request.FeedInfoRequest;
 import com.commafeed.frontend.model.request.FeedModificationRequest;
 import com.commafeed.frontend.model.request.IDRequest;
 import com.commafeed.frontend.model.request.MarkRequest;
+import com.commafeed.frontend.resource.CategoryREST;
 import com.commafeed.integration.BaseIT;
 import com.rometools.rome.feed.synd.SyndFeed;
 import com.rometools.rome.io.FeedException;
@@ -171,6 +172,7 @@ class FeedIT extends BaseIT {
 		@Test
 		void refreshAll() {
 			Long subscriptionId = subscribeAndWaitForEntries(getFeedUrl());
+			Assertions.assertEquals(2, getCategoryEntries(CategoryREST.ALL).getEntries().size());
 
 			// mariadb/mysql timestamp precision is 1 second
 			Instant threshold = Instant.now().minus(Duration.ofSeconds(1));
@@ -179,6 +181,7 @@ class FeedIT extends BaseIT {
 			Awaitility.await()
 					.atMost(Duration.ofSeconds(15))
 					.until(() -> getSubscription(subscriptionId), f -> f.getLastRefresh().isAfter(threshold));
+			Assertions.assertEquals(2, getCategoryEntries(CategoryREST.ALL).getEntries().size());
 
 			Assertions.assertEquals(HttpStatus.SC_TOO_MANY_REQUESTS, forceRefreshAllFeeds());
 		}

commafeed-server/src/test/java/com/commafeed/integration/rest/LargeDatasetIT.java

@@ -1,134 +0,0 @@
-package com.commafeed.integration.rest;
-
-import java.time.Duration;
-import java.time.Instant;
-
-import org.awaitility.Awaitility;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.mockserver.model.HttpRequest;
-import org.mockserver.model.HttpResponse;
-import org.mockserver.model.MediaType;
-
-import com.commafeed.TestConstants;
-import com.commafeed.frontend.model.Entries;
-import com.commafeed.frontend.model.Entry;
-import com.commafeed.frontend.model.request.MarkRequest;
-import com.commafeed.frontend.resource.CategoryREST;
-import com.commafeed.integration.BaseIT;
-
-import io.quarkus.test.junit.QuarkusTest;
-import io.restassured.RestAssured;
-import io.restassured.http.ContentType;
-
-@QuarkusTest
-class LargeDatasetIT extends BaseIT {
-
-	private static final int FEED_COUNT = 10;
-	private static final int ENTRIES_PER_FEED = 20;
-	private static final int TOTAL_ENTRIES = FEED_COUNT * ENTRIES_PER_FEED;
-
-	private Long firstSubscriptionId;
-
-	@BeforeEach
-	void setup() {
-		initialSetup(TestConstants.ADMIN_USERNAME, TestConstants.ADMIN_PASSWORD);
-		RestAssured.authentication = RestAssured.preemptive().basic(TestConstants.ADMIN_USERNAME, TestConstants.ADMIN_PASSWORD);
-
-		for (int i = 0; i < FEED_COUNT; i++) {
-			String path = "/feed/" + i;
-			getMockServerClient().when(HttpRequest.request().withMethod("GET").withPath(path))
-					.respond(HttpResponse.response().withBody(generateFeed(i)).withContentType(MediaType.APPLICATION_XML));
-			Long subscriptionId = subscribe("http://localhost:" + getMockServerClient().getPort() + path);
-			if (i == 0) {
-				firstSubscriptionId = subscriptionId;
-			}
-		}
-
-		Awaitility.await().atMost(Duration.ofSeconds(60)).until(() -> getAllEntries().getEntries().size(), count -> count >= TOTAL_ENTRIES);
-	}
-
-	@AfterEach
-	void cleanup() {
-		RestAssured.reset();
-	}
-
-	@Test
-	void markAllAsRead() {
-		Entries before = getAllEntries();
-		Assertions.assertEquals(TOTAL_ENTRIES, before.getEntries().size());
-		Assertions.assertTrue(before.getEntries().stream().noneMatch(Entry::isRead));
-
-		MarkRequest markRequest = new MarkRequest();
-		markRequest.setId(CategoryREST.ALL);
-		markRequest.setRead(true);
-		RestAssured.given().body(markRequest).contentType(ContentType.JSON).post("rest/category/mark").then().statusCode(200);
-
-		Entries after = getAllEntries();
-		Assertions.assertEquals(TOTAL_ENTRIES, after.getEntries().size());
-		Assertions.assertTrue(after.getEntries().stream().allMatch(Entry::isRead));
-	}
-
-	@Test
-	void refreshDoesNotCreateDuplicateEntries() {
-		Assertions.assertEquals(TOTAL_ENTRIES, getAllEntries().getEntries().size());
-		Instant threshold = Instant.now().minus(Duration.ofSeconds(1));
-		forceRefreshAllFeeds();
-
-		Awaitility.await()
-				.atMost(Duration.ofSeconds(15))
-				.until(() -> getSubscription(firstSubscriptionId), f -> f.getLastRefresh().isAfter(threshold));
-		Assertions.assertEquals(TOTAL_ENTRIES, getAllEntries().getEntries().size());
-	}
-
-	@Test
-	void paginationHasMore() {
-		Entries firstPage = RestAssured.given()
-				.get("rest/category/entries?id=all&readType=all&limit=20&offset=0")
-				.then()
-				.statusCode(200)
-				.extract()
-				.as(Entries.class);
-		Assertions.assertEquals(20, firstPage.getEntries().size());
-		Assertions.assertTrue(firstPage.isHasMore());
-
-		Entries lastPage = RestAssured.given()
-				.get("rest/category/entries?id=all&readType=all&limit=20&offset={offset}", TOTAL_ENTRIES - 20)
-				.then()
-				.statusCode(200)
-				.extract()
-				.as(Entries.class);
-		Assertions.assertEquals(20, lastPage.getEntries().size());
-		Assertions.assertFalse(lastPage.isHasMore());
-	}
-
-	private Entries getAllEntries() {
-		return RestAssured.given()
-				.get("rest/category/entries?id=all&readType=all&limit=1000")
-				.then()
-				.statusCode(200)
-				.extract()
-				.as(Entries.class);
-	}
-
-	private String generateFeed(int feedIndex) {
-		StringBuilder sb = new StringBuilder();
-		sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<rss version=\"2.0\">\n<channel>\n");
-		sb.append("<title>Feed ").append(feedIndex).append("</title>\n");
-		sb.append("<link>https://hostname.local/feed/").append(feedIndex).append("</link>\n");
-		sb.append("<description>Test feed ").append(feedIndex).append("</description>\n");
-		Instant base = Instant.parse("2024-01-01T00:00:00Z");
-		for (int i = 0; i < ENTRIES_PER_FEED; i++) {
-			sb.append("<item>\n");
-			sb.append("<title>Feed ").append(feedIndex).append(" Item ").append(i).append("</title>\n");
-			sb.append("<link>https://hostname.local/feed/").append(feedIndex).append("/item/").append(i).append("</link>\n");
-			sb.append("<description>Description for feed ").append(feedIndex).append(" item ").append(i).append("</description>\n");
-			sb.append("<pubDate>").append(base.minus(Duration.ofHours(i))).append("</pubDate>\n");
-			sb.append("</item>\n");
-		}
-		sb.append("</channel>\n</rss>");
-		return sb.toString();
-	}
-}