From 86edd54a218f46641a1b8324855c17fa9ddb867e Mon Sep 17 00:00:00 2001 From: Athou Date: Sun, 14 Apr 2013 18:28:48 +0200 Subject: [PATCH] index hashes --- .../com/commafeed/backend/dao/FeedDAO.java | 10 ++++++++-- .../commafeed/backend/dao/FeedEntryDAO.java | 19 +++++++++++++++++-- .../commafeed/backend/feeds/FeedParser.java | 2 ++ .../com/commafeed/backend/model/Feed.java | 18 ++++++++++++++++++ .../commafeed/backend/model/FeedEntry.java | 12 ++++++++++++ .../services/FeedSubscriptionService.java | 3 +++ 6 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/commafeed/backend/dao/FeedDAO.java b/src/main/java/com/commafeed/backend/dao/FeedDAO.java index 97941607..f000cf08 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedDAO.java @@ -10,6 +10,8 @@ import javax.persistence.criteria.CriteriaQuery; import javax.persistence.criteria.Predicate; import javax.persistence.criteria.Root; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.time.DateUtils; import com.commafeed.backend.model.Feed; @@ -51,8 +53,12 @@ public class FeedDAO extends GenericDAO { } public Feed findByUrl(String url) { - List feeds = findByField(Feed_.url, url); - return Iterables.getFirst(feeds, null); + List feeds = findByField(Feed_.urlHash, DigestUtils.sha1Hex(url)); + Feed feed = Iterables.getFirst(feeds, null); + if (feed != null && StringUtils.equals(url, feed.getUrl())) { + return feed; + } + return null; } public Feed findByIdWithEntries(Long feedId, int offset, int limit) { diff --git a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java index 80675925..8d3043e1 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java @@ -7,9 +7,12 @@ import javax.persistence.TypedQuery; import javax.persistence.criteria.CriteriaQuery; import javax.persistence.criteria.Root; +import org.apache.commons.codec.digest.DigestUtils; + import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntry_; +import com.google.api.client.util.Lists; import com.uaihebert.model.EasyCriteria; @Stateless @@ -17,11 +20,23 @@ import com.uaihebert.model.EasyCriteria; public class FeedEntryDAO extends GenericDAO { public List findByGuids(List guids) { + List hashes = Lists.newArrayList(); + for (String guid : guids) { + hashes.add(DigestUtils.sha1Hex(guid)); + } + EasyCriteria criteria = createCriteria(); criteria.setDistinctTrue(); - criteria.andStringIn(FeedEntry_.guid.getName(), guids); + criteria.andStringIn(FeedEntry_.guidHash.getName(), hashes); criteria.leftJoinFetch(FeedEntry_.feeds.getName()); - return criteria.getResultList(); + + List list = Lists.newArrayList(); + for (FeedEntry entry : criteria.getResultList()) { + if (guids.contains(entry.getGuid())) { + list.add(entry); + } + } + return list; } public List findByFeed(Feed feed, int offset, int limit) { diff --git a/src/main/java/com/commafeed/backend/feeds/FeedParser.java b/src/main/java/com/commafeed/backend/feeds/FeedParser.java index 0a1345b6..93fbf1c7 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedParser.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedParser.java @@ -5,6 +5,7 @@ import java.util.Calendar; import java.util.Date; import java.util.List; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.SystemUtils; import org.jsoup.Jsoup; @@ -49,6 +50,7 @@ public class FeedParser { for (SyndEntry item : items) { FeedEntry entry = new FeedEntry(); entry.setGuid(item.getUri()); + entry.setGuidHash(DigestUtils.sha1Hex(item.getUri())); entry.setUrl(item.getLink()); entry.setUpdated(getUpdateDate(item)); diff --git a/src/main/java/com/commafeed/backend/model/Feed.java b/src/main/java/com/commafeed/backend/model/Feed.java index 3ffbebb5..51ce05bd 100644 --- a/src/main/java/com/commafeed/backend/model/Feed.java +++ b/src/main/java/com/commafeed/backend/model/Feed.java @@ -21,12 +21,22 @@ import com.google.common.collect.Sets; @SuppressWarnings("serial") public class Feed extends AbstractModel { + /** + * The url of the feed + */ @Column(length = 2048, nullable = false) private String url; + @Column(length = 40, nullable = false) + @Index(name = "urlHash_index") + private String urlHash; + @Transient private String title; + /** + * The url of the website, extracted from the feed + */ @Column(length = 2048) private String link; @@ -129,4 +139,12 @@ public class Feed extends AbstractModel { this.disabledUntil = disabledUntil; } + public String getUrlHash() { + return urlHash; + } + + public void setUrlHash(String urlHash) { + this.urlHash = urlHash; + } + } diff --git a/src/main/java/com/commafeed/backend/model/FeedEntry.java b/src/main/java/com/commafeed/backend/model/FeedEntry.java index 01f221ac..62e2211b 100644 --- a/src/main/java/com/commafeed/backend/model/FeedEntry.java +++ b/src/main/java/com/commafeed/backend/model/FeedEntry.java @@ -28,6 +28,10 @@ public class FeedEntry extends AbstractModel { @Column(length = 2048, nullable = false) private String guid; + @Column(length = 40, nullable = false) + @Index(name = "guidHash_index") + private String guidHash; + @ManyToMany @JoinTable(name = "FEED_FEEDENTRIES", joinColumns = { @JoinColumn(name = "FEED_ID", nullable = false, updatable = false) }, inverseJoinColumns = { @JoinColumn(name = "FEEDENTRY_ID", nullable = false, updatable = false) }) private Set feeds = Sets.newHashSet(); @@ -106,4 +110,12 @@ public class FeedEntry extends AbstractModel { this.content = content; } + public String getGuidHash() { + return guidHash; + } + + public void setGuidHash(String guidHash) { + this.guidHash = guidHash; + } + } diff --git a/src/main/java/com/commafeed/backend/services/FeedSubscriptionService.java b/src/main/java/com/commafeed/backend/services/FeedSubscriptionService.java index def19be5..e00e8fc2 100644 --- a/src/main/java/com/commafeed/backend/services/FeedSubscriptionService.java +++ b/src/main/java/com/commafeed/backend/services/FeedSubscriptionService.java @@ -7,6 +7,8 @@ import javax.ejb.LockType; import javax.ejb.Singleton; import javax.inject.Inject; +import org.apache.commons.codec.digest.DigestUtils; + import com.commafeed.backend.dao.FeedDAO; import com.commafeed.backend.dao.FeedEntryDAO; import com.commafeed.backend.dao.FeedEntryStatusDAO; @@ -42,6 +44,7 @@ public class FeedSubscriptionService { if (feed == null) { feed = new Feed(); feed.setUrl(url); + feed.setUrlHash(DigestUtils.sha1Hex(url)); feedDAO.save(feed); }