garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

Make database cleaning batch size configurable

Browse Source
This commit is contained in:
D. Can Celasun
2023-07-11 14:21:59 +01:00
committed by Athou
parent 1af3dd452c
commit 6ce6b5ef0e
5 changed files with 30 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
commafeed-client/src/app/types.ts
View File

@@ -28,6 +28,7 @@ export interface ApplicationSettings {
queryTimeout: number
keepStatusDays: number
maxFeedCapacity: number
maxCleanupBatchSize: number
refreshIntervalMinutes: number
cache: ApplicationSettingsCache
announcement?: string

3
commafeed-server/config.dev.yml
View File

@@ -67,6 +67,9 @@ app:
# entries to keep per feed, old entries will be deleted, 0 to disable
maxFeedCapacity: 500
# rows to delete per query while cleaning up old entries, minimum 100
maxCleanupBatchSize: 100
# limit the number of feeds a user can subscribe to, 0 to disable
maxFeedsPerUser: 0

5
commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java
View File

@@ -135,6 +135,11 @@ public class CommaFeedConfiguration extends Configuration {
@Valid
private Integer maxFeedCapacity;
@NotNull
@Min(100)
@Valid
private Integer maxCleanupBatchSize;
@NotNull
@Valid
private Integer maxFeedsPerUser = 0;

26
commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java
View File

@@ -6,6 +6,7 @@ import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
@@ -14,7 +15,6 @@ import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
@@ -22,11 +22,10 @@ import lombok.extern.slf4j.Slf4j;
*
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class DatabaseCleaningService {
private static final int BATCH_SIZE = 100;
private final int batchSize;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
@@ -34,6 +33,17 @@ public class DatabaseCleaningService {
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
@Inject
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.getApplicationSettings().getMaxCleanupBatchSize();
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
@@ -44,7 +54,7 @@ public class DatabaseCleaningService {
for (Feed feed : feeds) {
long entriesDeleted;
do {
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), BATCH_SIZE));
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesTotal += entriesDeleted;
log.info("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
@@ -61,7 +71,7 @@ public class DatabaseCleaningService {
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE));
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.info("removed {} contents without entries", total);
} while (deleted != 0);
@@ -71,7 +81,7 @@ public class DatabaseCleaningService {
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
long total = 0;
while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE));
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
if (feeds.isEmpty()) {
break;
}
@@ -80,7 +90,7 @@ public class DatabaseCleaningService {
long remaining = feed.getCapacity() - maxFeedCapacity;
do {
final long rem = remaining;
int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(BATCH_SIZE, rem)));
int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(batchSize, rem)));
total += deleted;
remaining -= deleted;
log.info("removed {} entries for feeds exceeding capacity", total);
@@ -95,7 +105,7 @@ public class DatabaseCleaningService {
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, BATCH_SIZE));
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.info("removed {} old read statuses", total);
} while (deleted != 0);

3
commafeed-server/src/test/resources/config.test.yml
View File

@@ -67,6 +67,9 @@ app:
# entries to keep per feed, old entries will be deleted, 0 to disable
maxFeedCapacity: 500
# rows to delete per query while cleaning up old entries, minimum 100
maxCleanupBatchSize: 100
# limit the number of feeds a user can subscribe to, 0 to disable
maxFeedsPerUser: 0