diff --git a/commafeed-client/src/app/types.ts b/commafeed-client/src/app/types.ts index aa9e1c18..b7b7798f 100644 --- a/commafeed-client/src/app/types.ts +++ b/commafeed-client/src/app/types.ts @@ -3,38 +3,6 @@ export interface AddCategoryRequest { parentId?: string } -export interface ApplicationSettings { - publicUrl: string - allowRegistrations: boolean - createDemoAccount: boolean - googleAnalyticsTrackingCode?: string - googleAuthKey?: string - backgroundThreads: number - databaseUpdateThreads: number - smtpHost?: string - smtpPort?: number - smtpTls?: boolean - smtpUserName?: string - smtpPassword?: string - smtpFromAddress?: string - graphiteEnabled?: boolean - graphitePrefix?: string - graphiteHost?: string - graphitePort?: number - graphiteInterval?: number - heavyLoad: boolean - pubsubhubbub: boolean - imageProxyEnabled: boolean - queryTimeout: number - keepStatusDays: number - maxFeedCapacity: number - refreshIntervalMinutes: number - cache: ApplicationSettingsCache - announcement?: string - userAgent?: string - unreadThreshold?: Date -} - export interface Category { id: string parentId?: string @@ -300,8 +268,6 @@ export interface UserModel { admin: boolean } -export type ApplicationSettingsCache = "NOOP" | "REDIS" - export type ReadingMode = "all" | "unread" export type ReadingOrder = "asc" | "desc" diff --git a/commafeed-server/config.dev.yml b/commafeed-server/config.dev.yml index a27ed025..b631aaf6 100644 --- a/commafeed-server/config.dev.yml +++ b/commafeed-server/config.dev.yml @@ -27,7 +27,10 @@ app: # number of database updating threads databaseUpdateThreads: 1 - + + # rows to delete per query while cleaning up old entries + databaseCleanupBatchSize: 100 + # settings for sending emails (password recovery) smtpHost: localhost smtpPort: 25 diff --git a/commafeed-server/config.yml.example b/commafeed-server/config.yml.example index 298c02f7..e07e4a9c 100644 --- a/commafeed-server/config.yml.example +++ b/commafeed-server/config.yml.example @@ -28,6 +28,9 @@ app: # number of database updating threads databaseUpdateThreads: 1 + # rows to delete per query while cleaning up old entries + databaseCleanupBatchSize: 100 + # settings for sending emails (password recovery) smtpHost: smtpPort: diff --git a/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java b/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java index 229338e0..bbab016f 100644 --- a/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java +++ b/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java @@ -7,6 +7,7 @@ import javax.validation.Valid; import javax.validation.constraints.Min; import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; +import javax.validation.constraints.Positive; import org.apache.commons.lang3.time.DateUtils; @@ -95,6 +96,11 @@ public class CommaFeedConfiguration extends Configuration { @Valid private Integer databaseUpdateThreads; + @NotNull + @Positive + @Valid + private Integer databaseCleanupBatchSize = 100; + private String smtpHost; private int smtpPort; private boolean smtpTls; diff --git a/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java b/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java index 2155d60e..9e58d4fc 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java @@ -6,6 +6,7 @@ import java.util.List; import javax.inject.Inject; import javax.inject.Singleton; +import com.commafeed.CommaFeedConfiguration; import com.commafeed.backend.dao.FeedDAO; import com.commafeed.backend.dao.FeedEntryContentDAO; import com.commafeed.backend.dao.FeedEntryDAO; @@ -14,7 +15,6 @@ import com.commafeed.backend.dao.FeedEntryStatusDAO; import com.commafeed.backend.dao.UnitOfWork; import com.commafeed.backend.model.Feed; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; /** @@ -22,11 +22,10 @@ import lombok.extern.slf4j.Slf4j; * */ @Slf4j -@RequiredArgsConstructor(onConstructor = @__({ @Inject })) @Singleton public class DatabaseCleaningService { - private static final int BATCH_SIZE = 100; + private final int batchSize; private final UnitOfWork unitOfWork; private final FeedDAO feedDAO; @@ -34,6 +33,17 @@ public class DatabaseCleaningService { private final FeedEntryContentDAO feedEntryContentDAO; private final FeedEntryStatusDAO feedEntryStatusDAO; + @Inject + public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO, + FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO) { + this.unitOfWork = unitOfWork; + this.feedDAO = feedDAO; + this.feedEntryDAO = feedEntryDAO; + this.feedEntryContentDAO = feedEntryContentDAO; + this.feedEntryStatusDAO = feedEntryStatusDAO; + this.batchSize = config.getApplicationSettings().getDatabaseCleanupBatchSize(); + } + public void cleanFeedsWithoutSubscriptions() { log.info("cleaning feeds without subscriptions"); long total = 0; @@ -44,7 +54,7 @@ public class DatabaseCleaningService { for (Feed feed : feeds) { long entriesDeleted; do { - entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), BATCH_SIZE)); + entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize)); entriesTotal += entriesDeleted; log.info("removed {} entries for feeds without subscriptions", entriesTotal); } while (entriesDeleted > 0); @@ -61,7 +71,7 @@ public class DatabaseCleaningService { long total = 0; long deleted; do { - deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE)); + deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize)); total += deleted; log.info("removed {} contents without entries", total); } while (deleted != 0); @@ -71,7 +81,7 @@ public class DatabaseCleaningService { public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) { long total = 0; while (true) { - List feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE)); + List feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize)); if (feeds.isEmpty()) { break; } @@ -80,7 +90,7 @@ public class DatabaseCleaningService { long remaining = feed.getCapacity() - maxFeedCapacity; do { final long rem = remaining; - int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(BATCH_SIZE, rem))); + int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(batchSize, rem))); total += deleted; remaining -= deleted; log.info("removed {} entries for feeds exceeding capacity", total); @@ -95,7 +105,7 @@ public class DatabaseCleaningService { long total = 0; long deleted; do { - deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, BATCH_SIZE)); + deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize)); total += deleted; log.info("removed {} old read statuses", total); } while (deleted != 0); diff --git a/commafeed-server/src/test/resources/config.test.yml b/commafeed-server/src/test/resources/config.test.yml index 2d880cde..e5cb3ce6 100644 --- a/commafeed-server/src/test/resources/config.test.yml +++ b/commafeed-server/src/test/resources/config.test.yml @@ -28,6 +28,9 @@ app: # number of database updating threads databaseUpdateThreads: 1 + # rows to delete per query while cleaning up old entries + databaseCleanupBatchSize: 100 + # settings for sending emails (password recovery) smtpHost: localhost smtpPort: 25