fetch only ids to improve performance during cleanup

This commit is contained in:
Athou
2023-06-30 08:54:05 +02:00
parent 82cf0e154a
commit 922346bef6
5 changed files with 24 additions and 22 deletions

View File

@@ -28,13 +28,10 @@ public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
return query().select(content).from(content).where(content.contentHash.eq(contentHash), content.titleHash.eq(titleHash)).fetch(); return query().select(content).from(content).where(content.contentHash.eq(contentHash), content.titleHash.eq(titleHash)).fetch();
} }
public int deleteWithoutEntries(int max) { public long deleteWithoutEntries(int max) {
JPQLQuery<Integer> subQuery = JPAExpressions.selectOne().from(entry).where(entry.content.id.eq(content.id)); JPQLQuery<Integer> subQuery = JPAExpressions.selectOne().from(entry).where(entry.content.id.eq(content.id));
List<FeedEntryContent> list = query().selectFrom(content).where(subQuery.notExists()).limit(max).fetch(); List<Long> ids = query().select(content.id).from(content).where(subQuery.notExists()).limit(max).fetch();
int deleted = list.size(); return deleteQuery(content).where(content.id.in(ids)).execute();
delete(list);
return deleted;
} }
} }

View File

@@ -48,7 +48,6 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
} }
public int delete(Long feedId, long max) { public int delete(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).limit(max).fetch(); List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).limit(max).fetch();
return delete(list); return delete(list);
} }

View File

@@ -270,8 +270,13 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
return results; return results;
} }
public List<FeedEntryStatus> getOldStatuses(Date olderThan, int limit) { public long deleteOldStatuses(Date olderThan, int limit) {
return query().selectFrom(status).where(status.entryInserted.lt(olderThan), status.starred.isFalse()).limit(limit).fetch(); List<Long> ids = query().select(status.id)
.from(status)
.where(status.entryInserted.lt(olderThan), status.starred.isFalse())
.limit(limit)
.fetch();
return deleteQuery(status).where(status.id.in(ids)).execute();
} }
} }

View File

@@ -7,6 +7,7 @@ import org.hibernate.annotations.QueryHints;
import com.commafeed.backend.model.AbstractModel; import com.commafeed.backend.model.AbstractModel;
import com.querydsl.core.types.EntityPath; import com.querydsl.core.types.EntityPath;
import com.querydsl.jpa.impl.JPADeleteClause;
import com.querydsl.jpa.impl.JPAQuery; import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory; import com.querydsl.jpa.impl.JPAQueryFactory;
import com.querydsl.jpa.impl.JPAUpdateClause; import com.querydsl.jpa.impl.JPAUpdateClause;
@@ -30,6 +31,10 @@ public abstract class GenericDAO<T extends AbstractModel> extends AbstractDAO<T>
return new JPAUpdateClause(currentSession(), entityPath); return new JPAUpdateClause(currentSession(), entityPath);
} }
protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(currentSession(), entityPath);
}
public void saveOrUpdate(T model) { public void saveOrUpdate(T model) {
persist(model); persist(model);
} }

View File

@@ -34,15 +34,15 @@ public class DatabaseCleaningService {
private final FeedEntryContentDAO feedEntryContentDAO; private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO; private final FeedEntryStatusDAO feedEntryStatusDAO;
public long cleanFeedsWithoutSubscriptions() { public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions"); log.info("cleaning feeds without subscriptions");
long total = 0; long total = 0;
int deleted = 0; int deleted;
long entriesTotal = 0; long entriesTotal = 0;
do { do {
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1)); List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) { for (Feed feed : feeds) {
int entriesDeleted = 0; long entriesDeleted;
do { do {
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), BATCH_SIZE)); entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), BATCH_SIZE));
entriesTotal += entriesDeleted; entriesTotal += entriesDeleted;
@@ -54,23 +54,21 @@ public class DatabaseCleaningService {
log.info("removed {} feeds without subscriptions", total); log.info("removed {} feeds without subscriptions", total);
} while (deleted != 0); } while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total); log.info("cleanup done: {} feeds without subscriptions deleted", total);
return total;
} }
public long cleanContentsWithoutEntries() { public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries"); log.info("cleaning contents without entries");
long total = 0; long total = 0;
int deleted = 0; long deleted;
do { do {
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE)); deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE));
total += deleted; total += deleted;
log.info("removed {} contents without entries", total); log.info("removed {} contents without entries", total);
} while (deleted != 0); } while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total); log.info("cleanup done: {} contents without entries deleted", total);
return total;
} }
public long cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) { public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
long total = 0; long total = 0;
while (true) { while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE)); List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE));
@@ -90,19 +88,17 @@ public class DatabaseCleaningService {
} }
} }
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total); log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
return total;
} }
public long cleanStatusesOlderThan(final Date olderThan) { public void cleanStatusesOlderThan(final Date olderThan) {
log.info("cleaning old read statuses"); log.info("cleaning old read statuses");
long total = 0; long total = 0;
int deleted = 0; long deleted;
do { do {
deleted = unitOfWork.call(() -> feedEntryStatusDAO.delete(feedEntryStatusDAO.getOldStatuses(olderThan, BATCH_SIZE))); deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, BATCH_SIZE));
total += deleted; total += deleted;
log.info("removed {} old read statuses", total); log.info("removed {} old read statuses", total);
} while (deleted != 0); } while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total); log.info("cleanup done: {} old read statuses deleted", total);
return total;
} }
} }