diff --git a/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java b/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java index 0cf2962b..ab2df2ca 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java @@ -31,7 +31,7 @@ public class FeedEntryDAO extends GenericDAO { return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne(); } - public Set findExistingGuids(Feed feed, Set guidHashes) { + public Set findExistingGuidHashes(Set guidHashes, Feed feed) { if (guidHashes.isEmpty()) { return Set.of(); } diff --git a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java index 7440c3fa..c3a400d0 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java @@ -18,7 +18,6 @@ import org.apache.commons.lang3.StringUtils; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.commafeed.backend.Digests; -import com.commafeed.backend.dao.FeedEntryDAO; import com.commafeed.backend.dao.FeedSubscriptionDAO; import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.feed.parser.FeedParserResult.Content; @@ -43,7 +42,6 @@ public class FeedRefreshUpdater { private final UnitOfWork unitOfWork; private final FeedService feedService; private final FeedEntryService feedEntryService; - private final FeedEntryDAO feedEntryDAO; private final FeedSubscriptionDAO feedSubscriptionDAO; private final Striped locks; @@ -51,12 +49,11 @@ public class FeedRefreshUpdater { private final Meter feedUpdated; private final Meter entryInserted; - public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, FeedEntryDAO feedEntryDAO, - MetricRegistry metrics, FeedSubscriptionDAO feedSubscriptionDAO) { + public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics, + FeedSubscriptionDAO feedSubscriptionDAO) { this.unitOfWork = unitOfWork; this.feedService = feedService; this.feedEntryService = feedEntryService; - this.feedEntryDAO = feedEntryDAO; this.feedSubscriptionDAO = feedSubscriptionDAO; locks = Striped.lazyWeakLock(100000); @@ -129,18 +126,8 @@ public class FeedRefreshUpdater { Map> insertedUnreadEntriesBySubscription = new HashMap<>(); if (!entries.isEmpty()) { - Map entriesByGuidHash = new HashMap<>(); - for (Entry entry : entries) { - entriesByGuidHash.put(Digests.sha1Hex(entry.guid()), entry); - } - Set existingGuids = unitOfWork.call(() -> feedEntryDAO.findExistingGuids(feed, entriesByGuidHash.keySet())); - List newEntries = entriesByGuidHash.entrySet() - .stream() - .filter(e -> !existingGuids.contains(e.getKey())) - .map(Map.Entry::getValue) - .toList(); - List subscriptions = null; + List newEntries = unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries)); for (Entry entry : newEntries) { if (subscriptions == null) { subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed)); diff --git a/commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java b/commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java index 60bc7c77..a5f8ac8f 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java @@ -2,6 +2,8 @@ package com.commafeed.backend.service; import java.time.Instant; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import jakarta.inject.Singleton; @@ -52,6 +54,12 @@ public class FeedEntryService { return feedEntry; } + public List removeExistingEntries(Feed feed, List entries) { + Set guidHashes = entries.stream().map(e -> Digests.sha1Hex(e.guid())).collect(Collectors.toSet()); + Set existingGuidHashes = feedEntryDAO.findExistingGuidHashes(guidHashes, feed); + return entries.stream().filter(e -> !existingGuidHashes.contains(Digests.sha1Hex(e.guid()))).toList(); + } + public boolean applyFilter(FeedSubscription sub, FeedEntry entry) { boolean matches = true; try { diff --git a/commafeed-server/src/test/java/com/commafeed/integration/BaseIT.java b/commafeed-server/src/test/java/com/commafeed/integration/BaseIT.java index da5f41fd..0eb3fbb4 100644 --- a/commafeed-server/src/test/java/com/commafeed/integration/BaseIT.java +++ b/commafeed-server/src/test/java/com/commafeed/integration/BaseIT.java @@ -174,6 +174,15 @@ public abstract class BaseIT { .as(Entries.class); } + protected Entries getCategoryEntries(String categoryId, int offset, int limit) { + return RestAssured.given() + .get("rest/category/entries?id={id}&readType=all&offset={offset}&limit={limit}", categoryId, offset, limit) + .then() + .statusCode(HttpStatus.SC_OK) + .extract() + .as(Entries.class); + } + protected Entries getCategoryEntries(String categoryId, String keywords) { return RestAssured.given() .get("rest/category/entries?id={id}&readType=all&keywords={keywords}", categoryId, keywords) diff --git a/commafeed-server/src/test/java/com/commafeed/integration/rest/CategoryIT.java b/commafeed-server/src/test/java/com/commafeed/integration/rest/CategoryIT.java index 7aa7b5f0..a99ccac8 100644 --- a/commafeed-server/src/test/java/com/commafeed/integration/rest/CategoryIT.java +++ b/commafeed-server/src/test/java/com/commafeed/integration/rest/CategoryIT.java @@ -159,6 +159,19 @@ class CategoryIT extends BaseIT { Assertions.assertEquals(2, entries.getEntries().size()); } + @Test + void pagination() { + subscribeAndWaitForEntries(getFeedUrl()); + + Entries firstPage = getCategoryEntries(CategoryREST.ALL, 0, 1); + Assertions.assertEquals(1, firstPage.getEntries().size()); + Assertions.assertTrue(firstPage.isHasMore()); + + Entries lastPage = getCategoryEntries(CategoryREST.ALL, 1, 10); + Assertions.assertEquals(1, lastPage.getEntries().size()); + Assertions.assertFalse(lastPage.isHasMore()); + } + @Test void allAsFeed() throws FeedException { subscribeAndWaitForEntries(getFeedUrl()); diff --git a/commafeed-server/src/test/java/com/commafeed/integration/rest/FeedIT.java b/commafeed-server/src/test/java/com/commafeed/integration/rest/FeedIT.java index ae2f2ac3..b264d0e1 100644 --- a/commafeed-server/src/test/java/com/commafeed/integration/rest/FeedIT.java +++ b/commafeed-server/src/test/java/com/commafeed/integration/rest/FeedIT.java @@ -31,6 +31,7 @@ import com.commafeed.frontend.model.request.FeedInfoRequest; import com.commafeed.frontend.model.request.FeedModificationRequest; import com.commafeed.frontend.model.request.IDRequest; import com.commafeed.frontend.model.request.MarkRequest; +import com.commafeed.frontend.resource.CategoryREST; import com.commafeed.integration.BaseIT; import com.rometools.rome.feed.synd.SyndFeed; import com.rometools.rome.io.FeedException; @@ -171,6 +172,7 @@ class FeedIT extends BaseIT { @Test void refreshAll() { Long subscriptionId = subscribeAndWaitForEntries(getFeedUrl()); + Assertions.assertEquals(2, getCategoryEntries(CategoryREST.ALL).getEntries().size()); // mariadb/mysql timestamp precision is 1 second Instant threshold = Instant.now().minus(Duration.ofSeconds(1)); @@ -179,6 +181,7 @@ class FeedIT extends BaseIT { Awaitility.await() .atMost(Duration.ofSeconds(15)) .until(() -> getSubscription(subscriptionId), f -> f.getLastRefresh().isAfter(threshold)); + Assertions.assertEquals(2, getCategoryEntries(CategoryREST.ALL).getEntries().size()); Assertions.assertEquals(HttpStatus.SC_TOO_MANY_REQUESTS, forceRefreshAllFeeds()); } diff --git a/commafeed-server/src/test/java/com/commafeed/integration/rest/LargeDatasetIT.java b/commafeed-server/src/test/java/com/commafeed/integration/rest/LargeDatasetIT.java deleted file mode 100644 index 40105f53..00000000 --- a/commafeed-server/src/test/java/com/commafeed/integration/rest/LargeDatasetIT.java +++ /dev/null @@ -1,134 +0,0 @@ -package com.commafeed.integration.rest; - -import java.time.Duration; -import java.time.Instant; - -import org.awaitility.Awaitility; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockserver.model.HttpRequest; -import org.mockserver.model.HttpResponse; -import org.mockserver.model.MediaType; - -import com.commafeed.TestConstants; -import com.commafeed.frontend.model.Entries; -import com.commafeed.frontend.model.Entry; -import com.commafeed.frontend.model.request.MarkRequest; -import com.commafeed.frontend.resource.CategoryREST; -import com.commafeed.integration.BaseIT; - -import io.quarkus.test.junit.QuarkusTest; -import io.restassured.RestAssured; -import io.restassured.http.ContentType; - -@QuarkusTest -class LargeDatasetIT extends BaseIT { - - private static final int FEED_COUNT = 10; - private static final int ENTRIES_PER_FEED = 20; - private static final int TOTAL_ENTRIES = FEED_COUNT * ENTRIES_PER_FEED; - - private Long firstSubscriptionId; - - @BeforeEach - void setup() { - initialSetup(TestConstants.ADMIN_USERNAME, TestConstants.ADMIN_PASSWORD); - RestAssured.authentication = RestAssured.preemptive().basic(TestConstants.ADMIN_USERNAME, TestConstants.ADMIN_PASSWORD); - - for (int i = 0; i < FEED_COUNT; i++) { - String path = "/feed/" + i; - getMockServerClient().when(HttpRequest.request().withMethod("GET").withPath(path)) - .respond(HttpResponse.response().withBody(generateFeed(i)).withContentType(MediaType.APPLICATION_XML)); - Long subscriptionId = subscribe("http://localhost:" + getMockServerClient().getPort() + path); - if (i == 0) { - firstSubscriptionId = subscriptionId; - } - } - - Awaitility.await().atMost(Duration.ofSeconds(60)).until(() -> getAllEntries().getEntries().size(), count -> count >= TOTAL_ENTRIES); - } - - @AfterEach - void cleanup() { - RestAssured.reset(); - } - - @Test - void markAllAsRead() { - Entries before = getAllEntries(); - Assertions.assertEquals(TOTAL_ENTRIES, before.getEntries().size()); - Assertions.assertTrue(before.getEntries().stream().noneMatch(Entry::isRead)); - - MarkRequest markRequest = new MarkRequest(); - markRequest.setId(CategoryREST.ALL); - markRequest.setRead(true); - RestAssured.given().body(markRequest).contentType(ContentType.JSON).post("rest/category/mark").then().statusCode(200); - - Entries after = getAllEntries(); - Assertions.assertEquals(TOTAL_ENTRIES, after.getEntries().size()); - Assertions.assertTrue(after.getEntries().stream().allMatch(Entry::isRead)); - } - - @Test - void refreshDoesNotCreateDuplicateEntries() { - Assertions.assertEquals(TOTAL_ENTRIES, getAllEntries().getEntries().size()); - Instant threshold = Instant.now().minus(Duration.ofSeconds(1)); - forceRefreshAllFeeds(); - - Awaitility.await() - .atMost(Duration.ofSeconds(15)) - .until(() -> getSubscription(firstSubscriptionId), f -> f.getLastRefresh().isAfter(threshold)); - Assertions.assertEquals(TOTAL_ENTRIES, getAllEntries().getEntries().size()); - } - - @Test - void paginationHasMore() { - Entries firstPage = RestAssured.given() - .get("rest/category/entries?id=all&readType=all&limit=20&offset=0") - .then() - .statusCode(200) - .extract() - .as(Entries.class); - Assertions.assertEquals(20, firstPage.getEntries().size()); - Assertions.assertTrue(firstPage.isHasMore()); - - Entries lastPage = RestAssured.given() - .get("rest/category/entries?id=all&readType=all&limit=20&offset={offset}", TOTAL_ENTRIES - 20) - .then() - .statusCode(200) - .extract() - .as(Entries.class); - Assertions.assertEquals(20, lastPage.getEntries().size()); - Assertions.assertFalse(lastPage.isHasMore()); - } - - private Entries getAllEntries() { - return RestAssured.given() - .get("rest/category/entries?id=all&readType=all&limit=1000") - .then() - .statusCode(200) - .extract() - .as(Entries.class); - } - - private String generateFeed(int feedIndex) { - StringBuilder sb = new StringBuilder(); - sb.append("\n\n\n"); - sb.append("Feed ").append(feedIndex).append("\n"); - sb.append("https://hostname.local/feed/").append(feedIndex).append("\n"); - sb.append("Test feed ").append(feedIndex).append("\n"); - Instant base = Instant.parse("2024-01-01T00:00:00Z"); - for (int i = 0; i < ENTRIES_PER_FEED; i++) { - sb.append("\n"); - sb.append("Feed ").append(feedIndex).append(" Item ").append(i).append("\n"); - sb.append("https://hostname.local/feed/").append(feedIndex).append("/item/").append(i).append("\n"); - sb.append("Description for feed ").append(feedIndex).append(" item ").append(i).append("\n"); - sb.append("").append(base.minus(Duration.ofHours(i))).append("\n"); - sb.append("\n"); - } - sb.append("\n"); - return sb.toString(); - } -}