garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

Merge branch 'dcelasun-configurable-batch-size'

Browse Source
This commit is contained in:
Athou
2023-07-12 15:27:18 +02:00
parent 1af3dd452c 38c5beec2f
commit 45d0e0ec98
6 changed files with 34 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
commafeed-client/src/app/types.ts
View File

@@ -3,38 +3,6 @@ export interface AddCategoryRequest {
parentId?: string
}
export interface ApplicationSettings {
publicUrl: string
allowRegistrations: boolean
createDemoAccount: boolean
googleAnalyticsTrackingCode?: string
googleAuthKey?: string
backgroundThreads: number
databaseUpdateThreads: number
smtpHost?: string
smtpPort?: number
smtpTls?: boolean
smtpUserName?: string
smtpPassword?: string
smtpFromAddress?: string
graphiteEnabled?: boolean
graphitePrefix?: string
graphiteHost?: string
graphitePort?: number
graphiteInterval?: number
heavyLoad: boolean
pubsubhubbub: boolean
imageProxyEnabled: boolean
queryTimeout: number
keepStatusDays: number
maxFeedCapacity: number
refreshIntervalMinutes: number
cache: ApplicationSettingsCache
announcement?: string
userAgent?: string
unreadThreshold?: Date
}
export interface Category {
id: string
parentId?: string
@@ -300,8 +268,6 @@ export interface UserModel {
admin: boolean
}
export type ApplicationSettingsCache = "NOOP" | "REDIS"
export type ReadingMode = "all" | "unread"
export type ReadingOrder = "asc" | "desc"

5
commafeed-server/config.dev.yml
View File

@@ -27,7 +27,10 @@ app:
# number of database updating threads
databaseUpdateThreads: 1
# rows to delete per query while cleaning up old entries
databaseCleanupBatchSize: 100
# settings for sending emails (password recovery)
smtpHost: localhost
smtpPort: 25

3
commafeed-server/config.yml.example
View File

@@ -28,6 +28,9 @@ app:
# number of database updating threads
databaseUpdateThreads: 1
# rows to delete per query while cleaning up old entries
databaseCleanupBatchSize: 100
# settings for sending emails (password recovery)
smtpHost:
smtpPort:

6
commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java
View File

@@ -7,6 +7,7 @@ import javax.validation.Valid;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Positive;
import org.apache.commons.lang3.time.DateUtils;
@@ -95,6 +96,11 @@ public class CommaFeedConfiguration extends Configuration {
@Valid
private Integer databaseUpdateThreads;
@NotNull
@Positive
@Valid
private Integer databaseCleanupBatchSize = 100;
private String smtpHost;
private int smtpPort;
private boolean smtpTls;

26
commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java
View File

@@ -6,6 +6,7 @@ import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
@@ -14,7 +15,6 @@ import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
@@ -22,11 +22,10 @@ import lombok.extern.slf4j.Slf4j;
*
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class DatabaseCleaningService {
private static final int BATCH_SIZE = 100;
private final int batchSize;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
@@ -34,6 +33,17 @@ public class DatabaseCleaningService {
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
@Inject
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.getApplicationSettings().getDatabaseCleanupBatchSize();
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
@@ -44,7 +54,7 @@ public class DatabaseCleaningService {
for (Feed feed : feeds) {
long entriesDeleted;
do {
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), BATCH_SIZE));
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesTotal += entriesDeleted;
log.info("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
@@ -61,7 +71,7 @@ public class DatabaseCleaningService {
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE));
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.info("removed {} contents without entries", total);
} while (deleted != 0);
@@ -71,7 +81,7 @@ public class DatabaseCleaningService {
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
long total = 0;
while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE));
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
if (feeds.isEmpty()) {
break;
}
@@ -80,7 +90,7 @@ public class DatabaseCleaningService {
long remaining = feed.getCapacity() - maxFeedCapacity;
do {
final long rem = remaining;
int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(BATCH_SIZE, rem)));
int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(batchSize, rem)));
total += deleted;
remaining -= deleted;
log.info("removed {} entries for feeds exceeding capacity", total);
@@ -95,7 +105,7 @@ public class DatabaseCleaningService {
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, BATCH_SIZE));
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.info("removed {} old read statuses", total);
} while (deleted != 0);

3
commafeed-server/src/test/resources/config.test.yml
View File

@@ -28,6 +28,9 @@ app:
# number of database updating threads
databaseUpdateThreads: 1
# rows to delete per query while cleaning up old entries
databaseCleanupBatchSize: 100
# settings for sending emails (password recovery)
smtpHost: localhost
smtpPort: 25