diff --git a/commafeed-client/src/app/types.ts b/commafeed-client/src/app/types.ts index aa9e1c18..c31b3254 100644 --- a/commafeed-client/src/app/types.ts +++ b/commafeed-client/src/app/types.ts @@ -28,6 +28,7 @@ export interface ApplicationSettings { queryTimeout: number keepStatusDays: number maxFeedCapacity: number + maxCleanupBatchSize: number refreshIntervalMinutes: number cache: ApplicationSettingsCache announcement?: string diff --git a/commafeed-server/config.dev.yml b/commafeed-server/config.dev.yml index a27ed025..482a1b87 100644 --- a/commafeed-server/config.dev.yml +++ b/commafeed-server/config.dev.yml @@ -67,6 +67,9 @@ app: # entries to keep per feed, old entries will be deleted, 0 to disable maxFeedCapacity: 500 + # rows to delete per query while cleaning up old entries, minimum 100 + maxCleanupBatchSize: 100 + # limit the number of feeds a user can subscribe to, 0 to disable maxFeedsPerUser: 0 diff --git a/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java b/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java index 229338e0..6888bf34 100644 --- a/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java +++ b/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java @@ -135,6 +135,11 @@ public class CommaFeedConfiguration extends Configuration { @Valid private Integer maxFeedCapacity; + @NotNull + @Min(100) + @Valid + private Integer maxCleanupBatchSize; + @NotNull @Valid private Integer maxFeedsPerUser = 0; diff --git a/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java b/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java index 2155d60e..66d4c98d 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java @@ -6,6 +6,7 @@ import java.util.List; import javax.inject.Inject; import javax.inject.Singleton; +import com.commafeed.CommaFeedConfiguration; import com.commafeed.backend.dao.FeedDAO; import com.commafeed.backend.dao.FeedEntryContentDAO; import com.commafeed.backend.dao.FeedEntryDAO; @@ -14,7 +15,6 @@ import com.commafeed.backend.dao.FeedEntryStatusDAO; import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.model.Feed; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; /** @@ -22,11 +22,10 @@ import lombok.extern.slf4j.Slf4j; * */ @Slf4j -@RequiredArgsConstructor(onConstructor = @__({ @Inject })) @Singleton public class DatabaseCleaningService { - private static final int BATCH_SIZE = 100; + private final int batchSize; private final UnitOfWork unitOfWork; private final FeedDAO feedDAO; @@ -34,6 +33,17 @@ public class DatabaseCleaningService { private final FeedEntryContentDAO feedEntryContentDAO; private final FeedEntryStatusDAO feedEntryStatusDAO; + @Inject + public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO, + FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO) { + this.unitOfWork = unitOfWork; + this.feedDAO = feedDAO; + this.feedEntryDAO = feedEntryDAO; + this.feedEntryContentDAO = feedEntryContentDAO; + this.feedEntryStatusDAO = feedEntryStatusDAO; + this.batchSize = config.getApplicationSettings().getMaxCleanupBatchSize(); + } + public void cleanFeedsWithoutSubscriptions() { log.info("cleaning feeds without subscriptions"); long total = 0; @@ -44,7 +54,7 @@ public class DatabaseCleaningService { for (Feed feed : feeds) { long entriesDeleted; do { - entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), BATCH_SIZE)); + entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize)); entriesTotal += entriesDeleted; log.info("removed {} entries for feeds without subscriptions", entriesTotal); } while (entriesDeleted > 0); @@ -61,7 +71,7 @@ public class DatabaseCleaningService { long total = 0; long deleted; do { - deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE)); + deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize)); total += deleted; log.info("removed {} contents without entries", total); } while (deleted != 0); @@ -71,7 +81,7 @@ public class DatabaseCleaningService { public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) { long total = 0; while (true) { - List feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE)); + List feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize)); if (feeds.isEmpty()) { break; } @@ -80,7 +90,7 @@ public class DatabaseCleaningService { long remaining = feed.getCapacity() - maxFeedCapacity; do { final long rem = remaining; - int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(BATCH_SIZE, rem))); + int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(batchSize, rem))); total += deleted; remaining -= deleted; log.info("removed {} entries for feeds exceeding capacity", total); @@ -95,7 +105,7 @@ public class DatabaseCleaningService { long total = 0; long deleted; do { - deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, BATCH_SIZE)); + deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize)); total += deleted; log.info("removed {} old read statuses", total); } while (deleted != 0); diff --git a/commafeed-server/src/test/resources/config.test.yml b/commafeed-server/src/test/resources/config.test.yml index 2d880cde..9eb58d44 100644 --- a/commafeed-server/src/test/resources/config.test.yml +++ b/commafeed-server/src/test/resources/config.test.yml @@ -67,6 +67,9 @@ app: # entries to keep per feed, old entries will be deleted, 0 to disable maxFeedCapacity: 500 + # rows to delete per query while cleaning up old entries, minimum 100 + maxCleanupBatchSize: 100 + # limit the number of feeds a user can subscribe to, 0 to disable maxFeedsPerUser: 0