garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

add setting to delete old entries

Browse Source
This commit is contained in:
Athou
2024-01-07 20:43:19 +01:00
parent 039e91414e
commit 1292018de0
13 changed files with 104 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
commafeed-server/config.dev.yml
View File

@@ -67,6 +67,9 @@ app:
# entries to keep per feed, old entries will be deleted, 0 to disable
maxFeedCapacity: 500
# entries older than this will be deleted, 0 to disable
maxEntriesAgeDays: 365
# limit the number of feeds a user can subscribe to, 0 to disable
maxFeedsPerUser: 0

3
commafeed-server/config.yml.example
View File

@@ -67,6 +67,9 @@ app:
# entries to keep per feed, old entries will be deleted, 0 to disable
maxFeedCapacity: 500
# entries older than this will be deleted, 0 to disable
maxEntriesAgeDays: 365
# limit the number of feeds a user can subscribe to, 0 to disable
maxFeedsPerUser: 0

5
commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java
View File

@@ -146,6 +146,11 @@ public class CommaFeedConfiguration extends Configuration implements WebsocketBu
@Valid
private Integer maxFeedCapacity;
@NotNull
@Min(0)
@Valid
private Integer maxEntriesAgeDays = 0;
@NotNull
@Valid
private Integer maxFeedsPerUser = 0;

2
commafeed-server/src/main/java/com/commafeed/CommaFeedModule.java
View File

@@ -19,6 +19,7 @@ import com.commafeed.backend.favicon.DefaultFaviconFetcher;
import com.commafeed.backend.favicon.FacebookFaviconFetcher;
import com.commafeed.backend.favicon.YoutubeFaviconFetcher;
import com.commafeed.backend.task.DemoAccountCleanupTask;
import com.commafeed.backend.task.EntriesExceedingFeedCapacityCleanupTask;
import com.commafeed.backend.task.OldEntriesCleanupTask;
import com.commafeed.backend.task.OldStatusesCleanupTask;
import com.commafeed.backend.task.OrphanedContentsCleanupTask;
@@ -66,6 +67,7 @@ public class CommaFeedModule extends AbstractModule {
Multibinder<ScheduledTask> taskMultibinder = Multibinder.newSetBinder(binder(), ScheduledTask.class);
taskMultibinder.addBinding().to(OldStatusesCleanupTask.class);
taskMultibinder.addBinding().to(EntriesExceedingFeedCapacityCleanupTask.class);
taskMultibinder.addBinding().to(OldEntriesCleanupTask.class);
taskMultibinder.addBinding().to(OrphanedFeedsCleanupTask.class);
taskMultibinder.addBinding().to(OrphanedContentsCleanupTask.class);

12
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
View File

@@ -1,5 +1,6 @@
package com.commafeed.backend.dao;
import java.util.Date;
import java.util.List;
import org.hibernate.SessionFactory;
@@ -45,6 +46,17 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
return delete(list);
}
/**
* Delete entries older than a certain date
*/
public int deleteEntriesOlderThan(Date olderThan, long max) {
List<FeedEntry> list = query().selectFrom(entry).where(entry.updated.lt(olderThan)).orderBy(entry.updated.asc()).limit(max).fetch();
return delete(list);
}
/**
* Delete the oldest entries of a feed
*/
public int deleteOldEntries(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).orderBy(entry.updated.asc()).limit(max).fetch();
return delete(list);

11
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -1,10 +1,12 @@
package com.commafeed.backend.feed;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
@@ -47,12 +49,19 @@ public class FeedRefreshWorker {
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
List<Entry> entries = result.feed().entries();
Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
Integer maxEntriesAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays();
if (maxEntriesAgeDays > 0) {
Date threshold = DateUtils.addDays(new Date(), -1 * maxEntriesAgeDays);
entries = entries.stream().filter(entry -> entry.updated().after(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;

14
commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java
View File

@@ -83,6 +83,7 @@ public class DatabaseCleaningService {
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
@@ -105,6 +106,19 @@ public class DatabaseCleaningService {
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
public void cleanEntriesOlderThan(final Date olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.info("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Date olderThan) {
log.info("cleaning old read statuses");
long total = 0;

42
commafeed-server/src/main/java/com/commafeed/backend/task/EntriesExceedingFeedCapacityCleanupTask.java Normal file
View File

@@ -0,0 +1,42 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.DatabaseCleaningService;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

10
commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java
View File

@@ -1,7 +1,10 @@
package com.commafeed.backend.task;
import java.util.Date;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.time.DateUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.DatabaseCleaningService;
@@ -18,9 +21,10 @@ public class OldEntriesCleanupTask extends ScheduledTask {
@Override
public void run() {
int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
int maxAgeDays = config.getApplicationSettings().getMaxEntriesAgeDays();
if (maxAgeDays > 0) {
Date threshold = DateUtils.addDays(new Date(), -1 * maxAgeDays);
cleaner.cleanEntriesOlderThan(threshold);
}
}

2
commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java
View File

@@ -27,7 +27,7 @@ public class OldStatusesCleanupTask extends ScheduledTask {
@Override
public long getInitialDelay() {
return 10;
return 15;
}
@Override

2
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java
View File

@@ -21,7 +21,7 @@ public class OrphanedContentsCleanupTask extends ScheduledTask {
@Override
public long getInitialDelay() {
return 20;
return 25;
}
@Override

2
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java
View File

@@ -21,7 +21,7 @@ public class OrphanedFeedsCleanupTask extends ScheduledTask {
@Override
public long getInitialDelay() {
return 15;
return 20;
}
@Override

3
commafeed-server/src/test/resources/config.test.yml
View File

@@ -67,6 +67,9 @@ app:
# entries to keep per feed, old entries will be deleted, 0 to disable
maxFeedCapacity: 500
# entries older than this will be deleted, 0 to disable
maxEntriesAgeDays: 0
# limit the number of feeds a user can subscribe to, 0 to disable
maxFeedsPerUser: 0