From 379e342e30311365fd9471fadad737ad1c754640 Mon Sep 17 00:00:00 2001 From: Athou Date: Sun, 16 Jun 2013 11:48:23 +0200 Subject: [PATCH] added on-demand database cleanup (date is hardcoded for the moment) --- .../commafeed/backend/DatabaseCleaner.java | 29 +++++++++++++++++++ .../com/commafeed/backend/dao/FeedDAO.java | 2 +- .../commafeed/backend/dao/FeedEntryDAO.java | 9 ++++++ .../com/commafeed/backend/dao/GenericDAO.java | 6 ++-- .../backend/feeds/FeedRefreshWorker.java | 2 ++ .../com/commafeed/backend/model/Feed.java | 19 +++++++----- .../commafeed/backend/model/FeedEntry.java | 8 +---- .../com/commafeed/backend/model/User.java | 4 --- .../frontend/rest/resources/AbstractREST.java | 4 +++ .../frontend/rest/resources/AdminREST.java | 9 ++++++ .../resources/changelogs/db.changelog-1.1.xml | 22 ++++++++++++++ 11 files changed, 91 insertions(+), 23 deletions(-) create mode 100644 src/main/java/com/commafeed/backend/DatabaseCleaner.java diff --git a/src/main/java/com/commafeed/backend/DatabaseCleaner.java b/src/main/java/com/commafeed/backend/DatabaseCleaner.java new file mode 100644 index 00000000..ff0a905d --- /dev/null +++ b/src/main/java/com/commafeed/backend/DatabaseCleaner.java @@ -0,0 +1,29 @@ +package com.commafeed.backend; + +import java.util.Calendar; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import javax.ejb.Stateless; +import javax.inject.Inject; + +import com.commafeed.backend.dao.FeedEntryDAO; +import com.commafeed.backend.model.FeedEntry; + +@Stateless +public class DatabaseCleaner { + + @Inject + FeedEntryDAO feedEntryDAO; + + public void cleanOlderThan(long value, TimeUnit unit) { + Calendar cal = Calendar.getInstance(); + cal.add(Calendar.MINUTE, -1 * (int) unit.toMinutes(value)); + + List entries = feedEntryDAO.findByInserted(cal.getTime()); + for (FeedEntry entry : entries) { + feedEntryDAO.delete(entry.getStatuses()); + } + feedEntryDAO.delete(entries); + } +} diff --git a/src/main/java/com/commafeed/backend/dao/FeedDAO.java b/src/main/java/com/commafeed/backend/dao/FeedDAO.java index 13e5e2f6..a6f4645f 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedDAO.java @@ -90,6 +90,6 @@ public class FeedDAO extends GenericDAO { } public List findByTopic(String topic) { - return findByField(Feed_.pushTopic, topic); + return findByField(Feed_.pushTopicHash, DigestUtils.sha1Hex(topic)); } } diff --git a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java index 783b3274..d50ce1af 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java @@ -1,5 +1,6 @@ package com.commafeed.backend.dao; +import java.util.Date; import java.util.List; import javax.ejb.Stateless; @@ -55,6 +56,14 @@ public class FeedEntryDAO extends GenericDAO { return list; } + public List findByInserted(Date olderThan) { + CriteriaQuery query = builder.createQuery(getType()); + Root root = query.from(getType()); + + query.where(builder.lessThan(root.get(FeedEntry_.inserted), olderThan)); + return em.createQuery(query).getResultList(); + } + public List findByFeed(Feed feed, int offset, int limit) { CriteriaQuery query = builder.createQuery(getType()); Root root = query.from(getType()); diff --git a/src/main/java/com/commafeed/backend/dao/GenericDAO.java b/src/main/java/com/commafeed/backend/dao/GenericDAO.java index f747e975..f6b295c2 100644 --- a/src/main/java/com/commafeed/backend/dao/GenericDAO.java +++ b/src/main/java/com/commafeed/backend/dao/GenericDAO.java @@ -50,15 +50,15 @@ public abstract class GenericDAO { saveOrUpdate(Arrays.asList(models)); } - public void delete(T object) { + public void delete(AbstractModel object) { if (object != null) { object = em.merge(object); em.remove(object); } } - public void delete(List objects) { - for (T object : objects) { + public void delete(Collection objects) { + for (AbstractModel object : objects) { delete(object); } } diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java index 283c8366..a110d424 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java @@ -6,6 +6,7 @@ import java.util.List; import javax.inject.Inject; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.mutable.MutableBoolean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -164,6 +165,7 @@ public class FeedRefreshWorker { log.debug("feed {} has pubsub info: {}", feed.getUrl(), topic); feed.setPushHub(hub); feed.setPushTopic(topic); + feed.setPushTopicHash(DigestUtils.sha1Hex(topic)); } } diff --git a/src/main/java/com/commafeed/backend/model/Feed.java b/src/main/java/com/commafeed/backend/model/Feed.java index 3af09bd7..e0a4c016 100644 --- a/src/main/java/com/commafeed/backend/model/Feed.java +++ b/src/main/java/com/commafeed/backend/model/Feed.java @@ -11,14 +11,10 @@ import javax.persistence.Table; import javax.persistence.Temporal; import javax.persistence.TemporalType; -import org.hibernate.annotations.Index; - import com.google.common.collect.Sets; @Entity @Table(name = "FEEDS") -@org.hibernate.annotations.Table(appliesTo = "FEEDS", indexes = { @Index(name = "disabled_lastupdated_index", columnNames = { - "disabledUntil", "lastUpdated" }), }) @SuppressWarnings("serial") public class Feed extends AbstractModel { @@ -29,7 +25,6 @@ public class Feed extends AbstractModel { private String url; @Column(length = 40, nullable = false) - @Index(name = "urlHash_index") private String urlHash; /** @@ -42,7 +37,6 @@ public class Feed extends AbstractModel { * Last time we tried to fetch the feed */ @Temporal(TemporalType.TIMESTAMP) - @Index(name = "lastupdated_index") private Date lastUpdated; /** @@ -78,7 +72,6 @@ public class Feed extends AbstractModel { * feed refresh is disabled until this date */ @Temporal(TemporalType.TIMESTAMP) - @Index(name = "disableduntil_index") private Date disabledUntil; /** @@ -120,9 +113,11 @@ public class Feed extends AbstractModel { * detected topic for pubsubhubbub */ @Column(length = 2048) - @Index(name = "topic_index") private String pushTopic; + @Column(name = "push_topic_hash", length = 2048) + private String pushTopicHash; + /** * last time we subscribed for that topic on that hub */ @@ -289,4 +284,12 @@ public class Feed extends AbstractModel { this.lastEntryDate = lastEntryDate; } + public String getPushTopicHash() { + return pushTopicHash; + } + + public void setPushTopicHash(String pushTopicHash) { + this.pushTopicHash = pushTopicHash; + } + } diff --git a/src/main/java/com/commafeed/backend/model/FeedEntry.java b/src/main/java/com/commafeed/backend/model/FeedEntry.java index 6afd9b6c..7631840a 100644 --- a/src/main/java/com/commafeed/backend/model/FeedEntry.java +++ b/src/main/java/com/commafeed/backend/model/FeedEntry.java @@ -15,9 +15,6 @@ import javax.persistence.OneToOne; import javax.persistence.Table; import javax.persistence.Temporal; import javax.persistence.TemporalType; -import javax.persistence.Transient; - -import org.hibernate.annotations.Index; import com.google.api.client.util.Sets; @@ -30,7 +27,6 @@ public class FeedEntry extends AbstractModel { private String guid; @Column(length = 40, nullable = false) - @Index(name = "guidHash_index") private String guidHash; @ManyToMany @@ -44,15 +40,13 @@ public class FeedEntry extends AbstractModel { @Column(length = 2048) private String url; - //@Column(length = 128) - @Transient + @Column(name = "author", length = 128) private String author; @Temporal(TemporalType.TIMESTAMP) private Date inserted; @Temporal(TemporalType.TIMESTAMP) - @Index(name = "updated_index") private Date updated; @OneToMany(mappedBy = "entry") diff --git a/src/main/java/com/commafeed/backend/model/User.java b/src/main/java/com/commafeed/backend/model/User.java index 11dba1d4..d914a5a0 100644 --- a/src/main/java/com/commafeed/backend/model/User.java +++ b/src/main/java/com/commafeed/backend/model/User.java @@ -11,8 +11,6 @@ import javax.persistence.Table; import javax.persistence.Temporal; import javax.persistence.TemporalType; -import org.hibernate.annotations.Index; - import com.google.common.collect.Sets; @Entity @@ -21,11 +19,9 @@ import com.google.common.collect.Sets; public class User extends AbstractModel { @Column(length = 32, nullable = false, unique = true) - @Index(name = "username_index") private String name; @Column(length = 255, unique = true) - @Index(name = "useremail_index") private String email; @Column(length = 256, nullable = false) diff --git a/src/main/java/com/commafeed/frontend/rest/resources/AbstractREST.java b/src/main/java/com/commafeed/frontend/rest/resources/AbstractREST.java index 18ca3eab..b50fd4e0 100644 --- a/src/main/java/com/commafeed/frontend/rest/resources/AbstractREST.java +++ b/src/main/java/com/commafeed/frontend/rest/resources/AbstractREST.java @@ -24,6 +24,7 @@ import org.apache.wicket.protocol.http.servlet.ServletWebResponse; import org.apache.wicket.request.cycle.RequestCycle; import org.apache.wicket.util.crypt.Base64; +import com.commafeed.backend.DatabaseCleaner; import com.commafeed.backend.MetricsBean; import com.commafeed.backend.StartupBean; import com.commafeed.backend.dao.FeedCategoryDAO; @@ -124,6 +125,9 @@ public abstract class AbstractREST { @Inject FaviconFetcher faviconFetcher; + @Inject + DatabaseCleaner cleaner; + @PostConstruct public void init() { CommaFeedApplication app = CommaFeedApplication.get(); diff --git a/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java b/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java index 86ce239f..eccff5e0 100644 --- a/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java +++ b/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java @@ -2,6 +2,7 @@ package com.commafeed.frontend.rest.resources; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -189,4 +190,12 @@ public class AdminREST extends AbstractResourceREST { return Response.ok(map).build(); } + + @Path("/cleanup") + @GET + public Response cleanup() { + cleaner.cleanOlderThan(30, TimeUnit.DAYS); + return Response.ok("ok").build(); + } + } diff --git a/src/main/resources/changelogs/db.changelog-1.1.xml b/src/main/resources/changelogs/db.changelog-1.1.xml index e7a4c977..fb179220 100644 --- a/src/main/resources/changelogs/db.changelog-1.1.xml +++ b/src/main/resources/changelogs/db.changelog-1.1.xml @@ -186,6 +186,28 @@ + + + + + + + + + + + + + + + + + + + + + +