diff --git a/commafeed-server/config.dev.yml b/commafeed-server/config.dev.yml index e17c7df2..436cf710 100644 --- a/commafeed-server/config.dev.yml +++ b/commafeed-server/config.dev.yml @@ -67,6 +67,9 @@ app: # entries to keep per feed, old entries will be deleted, 0 to disable maxFeedCapacity: 500 + # entries older than this will be deleted, 0 to disable + maxEntriesAgeDays: 365 + # limit the number of feeds a user can subscribe to, 0 to disable maxFeedsPerUser: 0 diff --git a/commafeed-server/config.yml.example b/commafeed-server/config.yml.example index a4b0ae1d..28445c1f 100644 --- a/commafeed-server/config.yml.example +++ b/commafeed-server/config.yml.example @@ -67,6 +67,9 @@ app: # entries to keep per feed, old entries will be deleted, 0 to disable maxFeedCapacity: 500 + # entries older than this will be deleted, 0 to disable + maxEntriesAgeDays: 365 + # limit the number of feeds a user can subscribe to, 0 to disable maxFeedsPerUser: 0 diff --git a/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java b/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java index 1134081a..418190cc 100644 --- a/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java +++ b/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java @@ -146,6 +146,11 @@ public class CommaFeedConfiguration extends Configuration implements WebsocketBu @Valid private Integer maxFeedCapacity; + @NotNull + @Min(0) + @Valid + private Integer maxEntriesAgeDays = 0; + @NotNull @Valid private Integer maxFeedsPerUser = 0; diff --git a/commafeed-server/src/main/java/com/commafeed/CommaFeedModule.java b/commafeed-server/src/main/java/com/commafeed/CommaFeedModule.java index cfe3bc2c..945aa5b7 100644 --- a/commafeed-server/src/main/java/com/commafeed/CommaFeedModule.java +++ b/commafeed-server/src/main/java/com/commafeed/CommaFeedModule.java @@ -19,6 +19,7 @@ import com.commafeed.backend.favicon.DefaultFaviconFetcher; import com.commafeed.backend.favicon.FacebookFaviconFetcher; import com.commafeed.backend.favicon.YoutubeFaviconFetcher; import com.commafeed.backend.task.DemoAccountCleanupTask; +import com.commafeed.backend.task.EntriesExceedingFeedCapacityCleanupTask; import com.commafeed.backend.task.OldEntriesCleanupTask; import com.commafeed.backend.task.OldStatusesCleanupTask; import com.commafeed.backend.task.OrphanedContentsCleanupTask; @@ -66,6 +67,7 @@ public class CommaFeedModule extends AbstractModule { Multibinder taskMultibinder = Multibinder.newSetBinder(binder(), ScheduledTask.class); taskMultibinder.addBinding().to(OldStatusesCleanupTask.class); + taskMultibinder.addBinding().to(EntriesExceedingFeedCapacityCleanupTask.class); taskMultibinder.addBinding().to(OldEntriesCleanupTask.class); taskMultibinder.addBinding().to(OrphanedFeedsCleanupTask.class); taskMultibinder.addBinding().to(OrphanedContentsCleanupTask.class); diff --git a/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java b/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java index 34af1d88..bc914e07 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java @@ -1,5 +1,6 @@ package com.commafeed.backend.dao; +import java.util.Date; import java.util.List; import org.hibernate.SessionFactory; @@ -45,6 +46,17 @@ public class FeedEntryDAO extends GenericDAO { return delete(list); } + /** + * Delete entries older than a certain date + */ + public int deleteEntriesOlderThan(Date olderThan, long max) { + List list = query().selectFrom(entry).where(entry.updated.lt(olderThan)).orderBy(entry.updated.asc()).limit(max).fetch(); + return delete(list); + } + + /** + * Delete the oldest entries of a feed + */ public int deleteOldEntries(Long feedId, long max) { List list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).orderBy(entry.updated.asc()).limit(max).fetch(); return delete(list); diff --git a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java index 11a0ebe4..aa0491c6 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java @@ -1,10 +1,12 @@ package com.commafeed.backend.feed; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Optional; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; @@ -47,12 +49,19 @@ public class FeedRefreshWorker { feed.getLastPublishedDate(), feed.getLastContentHash()); // stops here if NotModifiedException or any other exception is thrown - Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity(); List entries = result.feed().entries(); + + Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity(); if (maxFeedCapacity > 0) { entries = entries.stream().limit(maxFeedCapacity).toList(); } + Integer maxEntriesAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays(); + if (maxEntriesAgeDays > 0) { + Date threshold = DateUtils.addDays(new Date(), -1 * maxEntriesAgeDays); + entries = entries.stream().filter(entry -> entry.updated().after(threshold)).toList(); + } + String urlAfterRedirect = result.urlAfterRedirect(); if (StringUtils.equals(url, urlAfterRedirect)) { urlAfterRedirect = null; diff --git a/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java b/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java index c1f90e3b..8da9ffd5 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java @@ -83,6 +83,7 @@ public class DatabaseCleaningService { } public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) { + log.info("cleaning entries exceeding feed capacity"); long total = 0; while (true) { List feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize)); @@ -105,6 +106,19 @@ public class DatabaseCleaningService { log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total); } + public void cleanEntriesOlderThan(final Date olderThan) { + log.info("cleaning old entries"); + long total = 0; + long deleted; + do { + deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize)); + entriesDeletedMeter.mark(deleted); + total += deleted; + log.info("removed {} old entries", total); + } while (deleted != 0); + log.info("cleanup done: {} old entries deleted", total); + } + public void cleanStatusesOlderThan(final Date olderThan) { log.info("cleaning old read statuses"); long total = 0; diff --git a/commafeed-server/src/main/java/com/commafeed/backend/task/EntriesExceedingFeedCapacityCleanupTask.java b/commafeed-server/src/main/java/com/commafeed/backend/task/EntriesExceedingFeedCapacityCleanupTask.java new file mode 100644 index 00000000..98b82191 --- /dev/null +++ b/commafeed-server/src/main/java/com/commafeed/backend/task/EntriesExceedingFeedCapacityCleanupTask.java @@ -0,0 +1,42 @@ +package com.commafeed.backend.task; + +import java.util.concurrent.TimeUnit; + +import com.commafeed.CommaFeedConfiguration; +import com.commafeed.backend.service.DatabaseCleaningService; + +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor(onConstructor = @__({ @Inject })) +@Singleton +public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask { + + private final CommaFeedConfiguration config; + private final DatabaseCleaningService cleaner; + + @Override + public void run() { + int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity(); + if (maxFeedCapacity > 0) { + cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity); + } + } + + @Override + public long getInitialDelay() { + return 10; + } + + @Override + public long getPeriod() { + return 60; + } + + @Override + public TimeUnit getTimeUnit() { + return TimeUnit.MINUTES; + } + +} diff --git a/commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java b/commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java index e3131722..963cb610 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java @@ -1,7 +1,10 @@ package com.commafeed.backend.task; +import java.util.Date; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.time.DateUtils; + import com.commafeed.CommaFeedConfiguration; import com.commafeed.backend.service.DatabaseCleaningService; @@ -18,9 +21,10 @@ public class OldEntriesCleanupTask extends ScheduledTask { @Override public void run() { - int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity(); - if (maxFeedCapacity > 0) { - cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity); + int maxAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays(); + if (maxAgeDays > 0) { + Date threshold = DateUtils.addDays(new Date(), -1 * maxAgeDays); + cleaner.cleanEntriesOlderThan(threshold); } } diff --git a/commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java b/commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java index ba843822..4edb58a2 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java @@ -27,7 +27,7 @@ public class OldStatusesCleanupTask extends ScheduledTask { @Override public long getInitialDelay() { - return 10; + return 15; } @Override diff --git a/commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java b/commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java index 4dacc185..aa5731f9 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java @@ -21,7 +21,7 @@ public class OrphanedContentsCleanupTask extends ScheduledTask { @Override public long getInitialDelay() { - return 20; + return 25; } @Override diff --git a/commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java b/commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java index 08912113..4405e49a 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java @@ -21,7 +21,7 @@ public class OrphanedFeedsCleanupTask extends ScheduledTask { @Override public long getInitialDelay() { - return 15; + return 20; } @Override diff --git a/commafeed-server/src/test/resources/config.test.yml b/commafeed-server/src/test/resources/config.test.yml index 70a63ae3..105241c2 100644 --- a/commafeed-server/src/test/resources/config.test.yml +++ b/commafeed-server/src/test/resources/config.test.yml @@ -67,6 +67,9 @@ app: # entries to keep per feed, old entries will be deleted, 0 to disable maxFeedCapacity: 500 + # entries older than this will be deleted, 0 to disable + maxEntriesAgeDays: 0 + # limit the number of feeds a user can subscribe to, 0 to disable maxFeedsPerUser: 0