diff --git a/.openshift/config.mysql.yml b/.openshift/config.mysql.yml index 04be9c88..dd7c5c39 100644 --- a/.openshift/config.mysql.yml +++ b/.openshift/config.mysql.yml @@ -44,6 +44,9 @@ app: # time to keep unread statuses (in days), 0 to disable keepStatusDays: 0 + # entries to keep per feed, old entries will be deleted, 0 to disable + maxFeedCapacity: 500 + # cache service to use, possible values are 'noop' and 'redis' cache: noop diff --git a/CHANGELOG b/CHANGELOG index 0f5c2e68..da98779c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,7 @@ v 2.1.0 - dropwizard upgrade to 0.8.0 - you have to remove the "app.contextPath" setting from your yml file, you can optionally use server.applicationContextPath instead + - new setting app.maxFeedCapacity for deleting old entries - ability to set filtering expressions for subscriptions to automatically mark new entries as read based on title, content, author or url. - ability to use !keyword or -keyword to exclude a keyword from a search query - facebook feeds now show user favicon instead of facebook favicon diff --git a/config.dev.yml b/config.dev.yml index b5da3d25..0e8accc0 100644 --- a/config.dev.yml +++ b/config.dev.yml @@ -44,6 +44,9 @@ app: # time to keep unread statuses (in days), 0 to disable keepStatusDays: 0 + # entries to keep per feed, old entries will be deleted, 0 to disable + maxFeedCapacity: 500 + # cache service to use, possible values are 'noop' and 'redis' cache: noop diff --git a/config.yml.example b/config.yml.example index 631b2300..a58e2205 100644 --- a/config.yml.example +++ b/config.yml.example @@ -48,6 +48,9 @@ app: # time to keep unread statuses (in days), 0 to disable keepStatusDays: 0 + # entries to keep per feed, old entries will be deleted, 0 to disable + maxFeedCapacity: 500 + # cache service to use, possible values are 'noop' and 'redis' cache: noop diff --git a/src/main/java/com/commafeed/CommaFeedConfiguration.java b/src/main/java/com/commafeed/CommaFeedConfiguration.java index 7ff5c3ac..cdcbfc88 100644 --- a/src/main/java/com/commafeed/CommaFeedConfiguration.java +++ b/src/main/java/com/commafeed/CommaFeedConfiguration.java @@ -110,6 +110,10 @@ public class CommaFeedConfiguration extends Configuration { @Min(0) private int keepStatusDays; + @NotNull + @Min(0) + private int maxFeedCapacity; + @NotNull @Min(0) private int refreshIntervalMinutes; diff --git a/src/main/java/com/commafeed/CommaFeedModule.java b/src/main/java/com/commafeed/CommaFeedModule.java index 5edd9feb..18260e27 100644 --- a/src/main/java/com/commafeed/CommaFeedModule.java +++ b/src/main/java/com/commafeed/CommaFeedModule.java @@ -15,6 +15,7 @@ import com.commafeed.backend.favicon.AbstractFaviconFetcher; import com.commafeed.backend.favicon.DefaultFaviconFetcher; import com.commafeed.backend.favicon.FacebookFaviconFetcher; import com.commafeed.backend.favicon.YoutubeFaviconFetcher; +import com.commafeed.backend.task.OldEntriesCleanupTask; import com.commafeed.backend.task.OldStatusesCleanupTask; import com.commafeed.backend.task.OrphansCleanupTask; import com.commafeed.backend.task.ScheduledTask; @@ -49,6 +50,7 @@ public class CommaFeedModule extends AbstractModule { Multibinder taskMultibinder = Multibinder.newSetBinder(binder(), ScheduledTask.class); taskMultibinder.addBinding().to(OldStatusesCleanupTask.class); + taskMultibinder.addBinding().to(OldEntriesCleanupTask.class); taskMultibinder.addBinding().to(OrphansCleanupTask.class); } } diff --git a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java index d5016419..4c3fedd7 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java @@ -1,11 +1,13 @@ package com.commafeed.backend.dao; -import java.util.Date; import java.util.List; import javax.inject.Inject; import javax.inject.Singleton; +import lombok.AllArgsConstructor; +import lombok.Getter; + import org.apache.commons.codec.digest.DigestUtils; import org.hibernate.SessionFactory; @@ -13,6 +15,9 @@ import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.QFeedEntry; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.mysema.query.Tuple; +import com.mysema.query.types.expr.NumberExpression; @Singleton public class FeedEntryDAO extends GenericDAO { @@ -30,10 +35,27 @@ public class FeedEntryDAO extends GenericDAO { return Iterables.getFirst(list, null); } - public int delete(Date olderThan, int max) { - List list = newQuery().from(entry).where(entry.inserted.lt(olderThan)).limit(max).list(entry); + public List findFeedsExceedingCapacity(long maxCapacity, long max) { + List list = Lists.newArrayList(); + NumberExpression count = entry.id.countDistinct(); + List tuples = newQuery().from(entry).groupBy(entry.feed).having(count.gt(maxCapacity)).limit(max).list(entry.feed.id, count); + for (Tuple tuple : tuples) { + list.add(new FeedCapacity(tuple.get(entry.feed.id), tuple.get(count))); + } + return list; + } + + public int deleteOldEntries(Long feedId, long max) { + List list = newQuery().from(entry).where(entry.feed.id.eq(feedId)).orderBy(entry.updated.asc()).limit(max).list(entry); int deleted = list.size(); delete(list); return deleted; } + + @AllArgsConstructor + @Getter + public static class FeedCapacity { + private Long id; + private Long capacity; + } } diff --git a/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java b/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java index 30418bbe..73eb09c0 100644 --- a/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java +++ b/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java @@ -1,9 +1,7 @@ package com.commafeed.backend.service; -import java.util.Calendar; import java.util.Date; import java.util.List; -import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.inject.Singleton; @@ -16,6 +14,7 @@ import org.hibernate.SessionFactory; import com.commafeed.backend.dao.FeedDAO; import com.commafeed.backend.dao.FeedEntryContentDAO; import com.commafeed.backend.dao.FeedEntryDAO; +import com.commafeed.backend.dao.FeedEntryDAO.FeedCapacity; import com.commafeed.backend.dao.FeedEntryStatusDAO; import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.model.Feed; @@ -75,23 +74,37 @@ public class DatabaseCleaningService { return total; } - public long cleanEntriesOlderThan(long value, TimeUnit unit) { - final Calendar cal = Calendar.getInstance(); - cal.add(Calendar.MINUTE, -1 * (int) unit.toMinutes(value)); - + public long cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) { long total = 0; - int deleted = 0; - do { - deleted = new UnitOfWork(sessionFactory) { + while (true) { + List feeds = new UnitOfWork>(sessionFactory) { @Override - protected Integer runInSession() throws Exception { - return feedEntryDAO.delete(cal.getTime(), BATCH_SIZE); + protected List runInSession() throws Exception { + return feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE); } }.run(); - total += deleted; - log.info("removed {} entries", total); - } while (deleted != 0); - log.info("cleanup done: {} entries deleted", total); + + if (feeds.isEmpty()) { + break; + } + + for (final FeedCapacity feed : feeds) { + long remaining = feed.getCapacity() - maxFeedCapacity; + do { + final long rem = remaining; + int deleted = new UnitOfWork(sessionFactory) { + @Override + protected Integer runInSession() throws Exception { + return feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(BATCH_SIZE, rem)); + }; + }.run(); + total += deleted; + remaining -= deleted; + log.info("removed {} entries for feeds exceeding capacity", total); + } while (remaining > 0); + } + } + log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total); return total; } diff --git a/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java b/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java new file mode 100644 index 00000000..9826b6d2 --- /dev/null +++ b/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java @@ -0,0 +1,43 @@ +package com.commafeed.backend.task; + +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import lombok.RequiredArgsConstructor; + +import com.commafeed.CommaFeedConfiguration; +import com.commafeed.backend.service.DatabaseCleaningService; + +@RequiredArgsConstructor(onConstructor = @__({ @Inject })) +@Singleton +public class OldEntriesCleanupTask extends ScheduledTask { + + private final CommaFeedConfiguration config; + private final DatabaseCleaningService cleaner; + + @Override + public void run() { + int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity(); + if (maxFeedCapacity > 0) { + cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity); + } + } + + @Override + public long getInitialDelay() { + return 5; + } + + @Override + public long getPeriod() { + return 60; + } + + @Override + public TimeUnit getTimeUnit() { + return TimeUnit.MINUTES; + } + +} diff --git a/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java b/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java index 5e7c1564..33a3e9fb 100644 --- a/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java +++ b/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java @@ -28,7 +28,7 @@ public class OldStatusesCleanupTask extends ScheduledTask { @Override public long getInitialDelay() { - return 10; + return 15; } @Override diff --git a/src/main/java/com/commafeed/backend/task/OrphansCleanupTask.java b/src/main/java/com/commafeed/backend/task/OrphansCleanupTask.java index 6f290dcf..6c11ec36 100644 --- a/src/main/java/com/commafeed/backend/task/OrphansCleanupTask.java +++ b/src/main/java/com/commafeed/backend/task/OrphansCleanupTask.java @@ -23,7 +23,7 @@ public class OrphansCleanupTask extends ScheduledTask { @Override public long getInitialDelay() { - return 5; + return 10; } @Override