Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-20 19:51:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor

Browse Source
This commit is contained in:
Athou
2026-04-10 14:33:33 +02:00
parent ec35d846cf
commit 6d2fb5acc2
7 changed files with 37 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
View File

@@ -31,7 +31,7 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
}
public Set<String> findExistingGuids(Feed feed, Set<String> guidHashes) {
public Set<String> findExistingGuidHashes(Set<String> guidHashes, Feed feed) {
if (guidHashes.isEmpty()) {
return Set.of();
}

19
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -18,7 +18,6 @@ import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
@@ -43,7 +42,6 @@ public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedEntryDAO feedEntryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final Striped<Lock> locks;
@@ -51,12 +49,11 @@ public class FeedRefreshUpdater {
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, FeedEntryDAO feedEntryDAO,
MetricRegistry metrics, FeedSubscriptionDAO feedSubscriptionDAO) {
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedEntryDAO = feedEntryDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
locks = Striped.lazyWeakLock(100000);
@@ -129,18 +126,8 @@ public class FeedRefreshUpdater {
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
Map<String, Entry> entriesByGuidHash = new HashMap<>();
for (Entry entry : entries) {
entriesByGuidHash.put(Digests.sha1Hex(entry.guid()), entry);
}
Set<String> existingGuids = unitOfWork.call(() -> feedEntryDAO.findExistingGuids(feed, entriesByGuidHash.keySet()));
List<Entry> newEntries = entriesByGuidHash.entrySet()
.stream()
.filter(e -> !existingGuids.contains(e.getKey()))
.map(Map.Entry::getValue)
.toList();
List<FeedSubscription> subscriptions = null;
List<Entry> newEntries = unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries));
for (Entry entry : newEntries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));

8
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java
View File

@@ -2,6 +2,8 @@ package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
@@ -52,6 +54,12 @@ public class FeedEntryService {
return feedEntry;
}
public List<Entry> removeExistingEntries(Feed feed, List<Entry> entries) {
Set<String> guidHashes = entries.stream().map(e -> Digests.sha1Hex(e.guid())).collect(Collectors.toSet());
Set<String> existingGuidHashes = feedEntryDAO.findExistingGuidHashes(guidHashes, feed);
return entries.stream().filter(e -> !existingGuidHashes.contains(Digests.sha1Hex(e.guid()))).toList();
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true;
try {

9
commafeed-server/src/test/java/com/commafeed/integration/BaseIT.java
View File

@@ -174,6 +174,15 @@ public abstract class BaseIT {
.as(Entries.class);
}
protected Entries getCategoryEntries(String categoryId, int offset, int limit) {
return RestAssured.given()
.get("rest/category/entries?id={id}&readType=all&offset={offset}&limit={limit}", categoryId, offset, limit)
.then()
.statusCode(HttpStatus.SC_OK)
.extract()
.as(Entries.class);
}
protected Entries getCategoryEntries(String categoryId, String keywords) {
return RestAssured.given()
.get("rest/category/entries?id={id}&readType=all&keywords={keywords}", categoryId, keywords)

13
commafeed-server/src/test/java/com/commafeed/integration/rest/CategoryIT.java
View File

@@ -159,6 +159,19 @@ class CategoryIT extends BaseIT {
Assertions.assertEquals(2, entries.getEntries().size());
}
@Test
void pagination() {
subscribeAndWaitForEntries(getFeedUrl());
Entries firstPage = getCategoryEntries(CategoryREST.ALL, 0, 1);
Assertions.assertEquals(1, firstPage.getEntries().size());
Assertions.assertTrue(firstPage.isHasMore());
Entries lastPage = getCategoryEntries(CategoryREST.ALL, 1, 10);
Assertions.assertEquals(1, lastPage.getEntries().size());
Assertions.assertFalse(lastPage.isHasMore());
}
@Test
void allAsFeed() throws FeedException {
subscribeAndWaitForEntries(getFeedUrl());

3
commafeed-server/src/test/java/com/commafeed/integration/rest/FeedIT.java
View File

@@ -31,6 +31,7 @@ import com.commafeed.frontend.model.request.FeedInfoRequest;
import com.commafeed.frontend.model.request.FeedModificationRequest;
import com.commafeed.frontend.model.request.IDRequest;
import com.commafeed.frontend.model.request.MarkRequest;
import com.commafeed.frontend.resource.CategoryREST;
import com.commafeed.integration.BaseIT;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.io.FeedException;
@@ -171,6 +172,7 @@ class FeedIT extends BaseIT {
@Test
void refreshAll() {
Long subscriptionId = subscribeAndWaitForEntries(getFeedUrl());
Assertions.assertEquals(2, getCategoryEntries(CategoryREST.ALL).getEntries().size());
// mariadb/mysql timestamp precision is 1 second
Instant threshold = Instant.now().minus(Duration.ofSeconds(1));
@@ -179,6 +181,7 @@ class FeedIT extends BaseIT {
Awaitility.await()
.atMost(Duration.ofSeconds(15))
.until(() -> getSubscription(subscriptionId), f -> f.getLastRefresh().isAfter(threshold));
Assertions.assertEquals(2, getCategoryEntries(CategoryREST.ALL).getEntries().size());
Assertions.assertEquals(HttpStatus.SC_TOO_MANY_REQUESTS, forceRefreshAllFeeds());
}

134
commafeed-server/src/test/java/com/commafeed/integration/rest/LargeDatasetIT.java
View File

@@ -1,134 +0,0 @@
package com.commafeed.integration.rest;
import java.time.Duration;
import java.time.Instant;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockserver.model.HttpRequest;
import org.mockserver.model.HttpResponse;
import org.mockserver.model.MediaType;
import com.commafeed.TestConstants;
import com.commafeed.frontend.model.Entries;
import com.commafeed.frontend.model.Entry;
import com.commafeed.frontend.model.request.MarkRequest;
import com.commafeed.frontend.resource.CategoryREST;
import com.commafeed.integration.BaseIT;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.RestAssured;
import io.restassured.http.ContentType;
@QuarkusTest
class LargeDatasetIT extends BaseIT {
private static final int FEED_COUNT = 10;
private static final int ENTRIES_PER_FEED = 20;
private static final int TOTAL_ENTRIES = FEED_COUNT * ENTRIES_PER_FEED;
private Long firstSubscriptionId;
@BeforeEach
void setup() {
initialSetup(TestConstants.ADMIN_USERNAME, TestConstants.ADMIN_PASSWORD);
RestAssured.authentication = RestAssured.preemptive().basic(TestConstants.ADMIN_USERNAME, TestConstants.ADMIN_PASSWORD);
for (int i = 0; i < FEED_COUNT; i++) {
String path = "/feed/" + i;
getMockServerClient().when(HttpRequest.request().withMethod("GET").withPath(path))
.respond(HttpResponse.response().withBody(generateFeed(i)).withContentType(MediaType.APPLICATION_XML));
Long subscriptionId = subscribe("http://localhost:" + getMockServerClient().getPort() + path);
if (i == 0) {
firstSubscriptionId = subscriptionId;
}
}
Awaitility.await().atMost(Duration.ofSeconds(60)).until(() -> getAllEntries().getEntries().size(), count -> count >= TOTAL_ENTRIES);
}
@AfterEach
void cleanup() {
RestAssured.reset();
}
@Test
void markAllAsRead() {
Entries before = getAllEntries();
Assertions.assertEquals(TOTAL_ENTRIES, before.getEntries().size());
Assertions.assertTrue(before.getEntries().stream().noneMatch(Entry::isRead));
MarkRequest markRequest = new MarkRequest();
markRequest.setId(CategoryREST.ALL);
markRequest.setRead(true);
RestAssured.given().body(markRequest).contentType(ContentType.JSON).post("rest/category/mark").then().statusCode(200);
Entries after = getAllEntries();
Assertions.assertEquals(TOTAL_ENTRIES, after.getEntries().size());
Assertions.assertTrue(after.getEntries().stream().allMatch(Entry::isRead));
}
@Test
void refreshDoesNotCreateDuplicateEntries() {
Assertions.assertEquals(TOTAL_ENTRIES, getAllEntries().getEntries().size());
Instant threshold = Instant.now().minus(Duration.ofSeconds(1));
forceRefreshAllFeeds();
Awaitility.await()
.atMost(Duration.ofSeconds(15))
.until(() -> getSubscription(firstSubscriptionId), f -> f.getLastRefresh().isAfter(threshold));
Assertions.assertEquals(TOTAL_ENTRIES, getAllEntries().getEntries().size());
}
@Test
void paginationHasMore() {
Entries firstPage = RestAssured.given()
.get("rest/category/entries?id=all&readType=all&limit=20&offset=0")
.then()
.statusCode(200)
.extract()
.as(Entries.class);
Assertions.assertEquals(20, firstPage.getEntries().size());
Assertions.assertTrue(firstPage.isHasMore());
Entries lastPage = RestAssured.given()
.get("rest/category/entries?id=all&readType=all&limit=20&offset={offset}", TOTAL_ENTRIES - 20)
.then()
.statusCode(200)
.extract()
.as(Entries.class);
Assertions.assertEquals(20, lastPage.getEntries().size());
Assertions.assertFalse(lastPage.isHasMore());
}
private Entries getAllEntries() {
return RestAssured.given()
.get("rest/category/entries?id=all&readType=all&limit=1000")
.then()
.statusCode(200)
.extract()
.as(Entries.class);
}
private String generateFeed(int feedIndex) {
StringBuilder sb = new StringBuilder();
sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<rss version=\"2.0\">\n<channel>\n");
sb.append("<title>Feed ").append(feedIndex).append("</title>\n");
sb.append("<link>https://hostname.local/feed/").append(feedIndex).append("</link>\n");
sb.append("<description>Test feed ").append(feedIndex).append("</description>\n");
Instant base = Instant.parse("2024-01-01T00:00:00Z");
for (int i = 0; i < ENTRIES_PER_FEED; i++) {
sb.append("<item>\n");
sb.append("<title>Feed ").append(feedIndex).append(" Item ").append(i).append("</title>\n");
sb.append("<link>https://hostname.local/feed/").append(feedIndex).append("/item/").append(i).append("</link>\n");
sb.append("<description>Description for feed ").append(feedIndex).append(" item ").append(i).append("</description>\n");
sb.append("<pubDate>").append(base.minus(Duration.ofHours(i))).append("</pubDate>\n");
sb.append("</item>\n");
}
sb.append("</channel>\n</rss>");
return sb.toString();
}
}