add setting to delete old entries

commafeed-server/config.dev.yml

@@ -67,6 +67,9 @@ app:
   # entries to keep per feed, old entries will be deleted, 0 to disable
   maxFeedCapacity: 500
 
+  # entries older than this will be deleted, 0 to disable
+  maxEntriesAgeDays: 365
+
   # limit the number of feeds a user can subscribe to, 0 to disable
   maxFeedsPerUser: 0
 

commafeed-server/config.yml.example

@@ -67,6 +67,9 @@ app:
   # entries to keep per feed, old entries will be deleted, 0 to disable
   maxFeedCapacity: 500
 
+  # entries older than this will be deleted, 0 to disable
+  maxEntriesAgeDays: 365
+
   # limit the number of feeds a user can subscribe to, 0 to disable
   maxFeedsPerUser: 0
 

commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java

@@ -146,6 +146,11 @@ public class CommaFeedConfiguration extends Configuration implements WebsocketBu
 		@Valid
 		private Integer maxFeedCapacity;
 
+		@NotNull
+		@Min(0)
+		@Valid
+		private Integer maxEntriesAgeDays = 0;
+
 		@NotNull
 		@Valid
 		private Integer maxFeedsPerUser = 0;

commafeed-server/src/main/java/com/commafeed/CommaFeedModule.java

@@ -19,6 +19,7 @@ import com.commafeed.backend.favicon.DefaultFaviconFetcher;
 import com.commafeed.backend.favicon.FacebookFaviconFetcher;
 import com.commafeed.backend.favicon.YoutubeFaviconFetcher;
 import com.commafeed.backend.task.DemoAccountCleanupTask;
+import com.commafeed.backend.task.EntriesExceedingFeedCapacityCleanupTask;
 import com.commafeed.backend.task.OldEntriesCleanupTask;
 import com.commafeed.backend.task.OldStatusesCleanupTask;
 import com.commafeed.backend.task.OrphanedContentsCleanupTask;
@@ -66,6 +67,7 @@ public class CommaFeedModule extends AbstractModule {
 
 		Multibinder<ScheduledTask> taskMultibinder = Multibinder.newSetBinder(binder(), ScheduledTask.class);
 		taskMultibinder.addBinding().to(OldStatusesCleanupTask.class);
+		taskMultibinder.addBinding().to(EntriesExceedingFeedCapacityCleanupTask.class);
 		taskMultibinder.addBinding().to(OldEntriesCleanupTask.class);
 		taskMultibinder.addBinding().to(OrphanedFeedsCleanupTask.class);
 		taskMultibinder.addBinding().to(OrphanedContentsCleanupTask.class);

commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java

@@ -1,5 +1,6 @@
 package com.commafeed.backend.dao;
 
+import java.util.Date;
 import java.util.List;
 
 import org.hibernate.SessionFactory;
@@ -45,6 +46,17 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
 		return delete(list);
 	}
 
+	/**
+	 * Delete entries older than a certain date
+	 */
+	public int deleteEntriesOlderThan(Date olderThan, long max) {
+		List<FeedEntry> list = query().selectFrom(entry).where(entry.updated.lt(olderThan)).orderBy(entry.updated.asc()).limit(max).fetch();
+		return delete(list);
+	}
+
+	/**
+	 * Delete the oldest entries of a feed
+	 */
 	public int deleteOldEntries(Long feedId, long max) {
 		List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).orderBy(entry.updated.asc()).limit(max).fetch();
 		return delete(list);

commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java

@@ -1,10 +1,12 @@
 package com.commafeed.backend.feed;
 
 import java.util.Collections;
+import java.util.Date;
 import java.util.List;
 import java.util.Optional;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
 
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.MetricRegistry;
@@ -47,12 +49,19 @@ public class FeedRefreshWorker {
 					feed.getLastPublishedDate(), feed.getLastContentHash());
 			// stops here if NotModifiedException or any other exception is thrown
 
-			Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
 			List<Entry> entries = result.feed().entries();
+
+			Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
 			if (maxFeedCapacity > 0) {
 				entries = entries.stream().limit(maxFeedCapacity).toList();
 			}
 
+			Integer maxEntriesAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays();
+			if (maxEntriesAgeDays > 0) {
+				Date threshold = DateUtils.addDays(new Date(), -1 * maxEntriesAgeDays);
+				entries = entries.stream().filter(entry -> entry.updated().after(threshold)).toList();
+			}
+
 			String urlAfterRedirect = result.urlAfterRedirect();
 			if (StringUtils.equals(url, urlAfterRedirect)) {
 				urlAfterRedirect = null;

commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java

@@ -83,6 +83,7 @@ public class DatabaseCleaningService {
 	}
 
 	public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
+		log.info("cleaning entries exceeding feed capacity");
 		long total = 0;
 		while (true) {
 			List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
@@ -105,6 +106,19 @@ public class DatabaseCleaningService {
 		log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
 	}
 
+	public void cleanEntriesOlderThan(final Date olderThan) {
+		log.info("cleaning old entries");
+		long total = 0;
+		long deleted;
+		do {
+			deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize));
+			entriesDeletedMeter.mark(deleted);
+			total += deleted;
+			log.info("removed {} old entries", total);
+		} while (deleted != 0);
+		log.info("cleanup done: {} old entries deleted", total);
+	}
+
 	public void cleanStatusesOlderThan(final Date olderThan) {
 		log.info("cleaning old read statuses");
 		long total = 0;

commafeed-server/src/main/java/com/commafeed/backend/task/EntriesExceedingFeedCapacityCleanupTask.java

@@ -0,0 +1,42 @@
+package com.commafeed.backend.task;
+
+import java.util.concurrent.TimeUnit;
+
+import com.commafeed.CommaFeedConfiguration;
+import com.commafeed.backend.service.DatabaseCleaningService;
+
+import jakarta.inject.Inject;
+import jakarta.inject.Singleton;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@Singleton
+public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
+
+	private final CommaFeedConfiguration config;
+	private final DatabaseCleaningService cleaner;
+
+	@Override
+	public void run() {
+		int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
+		if (maxFeedCapacity > 0) {
+			cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
+		}
+	}
+
+	@Override
+	public long getInitialDelay() {
+		return 10;
+	}
+
+	@Override
+	public long getPeriod() {
+		return 60;
+	}
+
+	@Override
+	public TimeUnit getTimeUnit() {
+		return TimeUnit.MINUTES;
+	}
+
+}

commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java

@@ -1,7 +1,10 @@
 package com.commafeed.backend.task;
 
+import java.util.Date;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.commons.lang3.time.DateUtils;
+
 import com.commafeed.CommaFeedConfiguration;
 import com.commafeed.backend.service.DatabaseCleaningService;
 
@@ -18,9 +21,10 @@ public class OldEntriesCleanupTask extends ScheduledTask {
 
 	@Override
 	public void run() {
-		int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
+		int maxAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays();
-		if (maxFeedCapacity > 0) {
+		if (maxAgeDays > 0) {
-			cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
+			Date threshold = DateUtils.addDays(new Date(), -1 * maxAgeDays);
+			cleaner.cleanEntriesOlderThan(threshold);
 		}
 	}
 

commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java

@@ -27,7 +27,7 @@ public class OldStatusesCleanupTask extends ScheduledTask {
 
 	@Override
 	public long getInitialDelay() {
-		return 10;
+		return 15;
 	}
 
 	@Override

commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java

@@ -21,7 +21,7 @@ public class OrphanedContentsCleanupTask extends ScheduledTask {
 
 	@Override
 	public long getInitialDelay() {
-		return 20;
+		return 25;
 	}
 
 	@Override

commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java

@@ -21,7 +21,7 @@ public class OrphanedFeedsCleanupTask extends ScheduledTask {
 
 	@Override
 	public long getInitialDelay() {
-		return 15;
+		return 20;
 	}
 
 	@Override

commafeed-server/src/test/resources/config.test.yml

@@ -67,6 +67,9 @@ app:
   # entries to keep per feed, old entries will be deleted, 0 to disable
   maxFeedCapacity: 500
 
+  # entries older than this will be deleted, 0 to disable
+  maxEntriesAgeDays: 0
+
   # limit the number of feeds a user can subscribe to, 0 to disable
   maxFeedsPerUser: 0