diff --git a/src/main/java/com/commafeed/backend/DatabaseCleaner.java b/src/main/java/com/commafeed/backend/DatabaseCleaner.java index fc5c036d..f4eeba81 100644 --- a/src/main/java/com/commafeed/backend/DatabaseCleaner.java +++ b/src/main/java/com/commafeed/backend/DatabaseCleaner.java @@ -10,6 +10,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.commafeed.backend.dao.FeedDAO; +import com.commafeed.backend.dao.FeedEntryContentDAO; import com.commafeed.backend.dao.FeedEntryDAO; import com.commafeed.backend.dao.FeedSubscriptionDAO; import com.commafeed.backend.model.Feed; @@ -28,6 +29,9 @@ public class DatabaseCleaner { @Inject FeedSubscriptionDAO feedSubscriptionDAO; + + @Inject + FeedEntryContentDAO feedEntryContentDAO; @Inject ApplicationSettingsService applicationSettingsService; @@ -45,16 +49,16 @@ public class DatabaseCleaner { return total; } - public long cleanEntriesWithoutFeeds() { + public long cleanContentsWithoutEntries() { long total = 0; int deleted = -1; do { - deleted = feedEntryDAO.deleteWithoutFeeds(100); + deleted = feedEntryContentDAO.deleteWithoutEntries(10); total += deleted; - log.info("removed {} entries without feeds", total); + log.info("removed {} feeds without subscriptions", total); } while (deleted != 0); - log.info("cleanup done: {} entries without feeds deleted", total); + log.info("cleanup done: {} feeds without subscriptions deleted", total); return total; } diff --git a/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java b/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java new file mode 100644 index 00000000..db613b62 --- /dev/null +++ b/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java @@ -0,0 +1,66 @@ +package com.commafeed.backend.dao; + +import java.util.List; + +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Join; +import javax.persistence.criteria.JoinType; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; + +import org.apache.commons.codec.digest.DigestUtils; + +import com.commafeed.backend.model.FeedEntry; +import com.commafeed.backend.model.FeedEntryContent; +import com.commafeed.backend.model.FeedEntryContent_; +import com.commafeed.backend.model.FeedEntry_; +import com.google.common.collect.Iterables; + +public class FeedEntryContentDAO extends GenericDAO { + + public FeedEntryContent findExisting(FeedEntryContent content) { + + CriteriaQuery query = builder.createQuery(getType()); + Root root = query.from(getType()); + + Predicate p1 = builder.equal(root.get(FeedEntryContent_.contentHash), + DigestUtils.sha1Hex(content.getContent())); + Predicate p2 = null; + if (content.getTitle() == null) { + p2 = builder.isNull(root.get(FeedEntryContent_.title)); + } else { + p2 = builder.equal(root.get(FeedEntryContent_.title), + content.getTitle()); + } + + Predicate p3 = null; + if (content.getAuthor() == null) { + p3 = builder.isNull(root.get(FeedEntryContent_.author)); + } else { + p3 = builder.equal(root.get(FeedEntryContent_.author), + content.getAuthor()); + } + + query.where(p1, p2, p3); + TypedQuery q = em.createQuery(query); + return Iterables.getFirst(q.getResultList(), null); + + } + + public int deleteWithoutEntries(int max) { + CriteriaQuery query = builder.createQuery(getType()); + Root root = query.from(getType()); + + Join join = root.join( + FeedEntryContent_.entries, JoinType.LEFT); + query.where(builder.isNull(join.get(FeedEntry_.id))); + TypedQuery q = em.createQuery(query); + q.setMaxResults(max); + + List list = q.getResultList(); + int deleted = list.size(); + return deleted; + + } +} diff --git a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java index 77c28006..a8d607ee 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java @@ -7,19 +7,16 @@ import javax.ejb.Stateless; import javax.inject.Inject; import javax.persistence.TypedQuery; import javax.persistence.criteria.CriteriaQuery; -import javax.persistence.criteria.JoinType; +import javax.persistence.criteria.Predicate; import javax.persistence.criteria.Root; -import javax.persistence.criteria.SetJoin; import org.apache.commons.codec.digest.DigestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntry_; -import com.commafeed.backend.model.FeedFeedEntry; -import com.commafeed.backend.model.FeedFeedEntry_; +import com.commafeed.backend.model.Feed_; import com.commafeed.backend.services.ApplicationSettingsService; import com.google.common.collect.Iterables; @@ -32,49 +29,21 @@ public class FeedEntryDAO extends GenericDAO { protected static final Logger log = LoggerFactory .getLogger(FeedEntryDAO.class); - public static class EntryWithFeed { - public FeedEntry entry; - public FeedFeedEntry ffe; + public FeedEntry findExisting(String guid, String url, Long feedId) { - public EntryWithFeed(FeedEntry entry, FeedFeedEntry ffe) { - this.entry = entry; - this.ffe = ffe; - } - } - - public EntryWithFeed findExisting(String guid, String url, Long feedId) { - - TypedQuery q = em.createNamedQuery( - "EntryStatus.existing", EntryWithFeed.class); - q.setParameter("guidHash", DigestUtils.sha1Hex(guid)); - q.setParameter("url", url); - q.setParameter("feedId", feedId); - - EntryWithFeed result = null; - List list = q.getResultList(); - for (EntryWithFeed ewf : list) { - if (ewf.entry != null && ewf.ffe != null) { - result = ewf; - break; - } - } - if (result == null) { - result = Iterables.getFirst(list, null); - } - return result; - } - - public List findByFeed(Feed feed, int offset, int limit) { CriteriaQuery query = builder.createQuery(getType()); Root root = query.from(getType()); - SetJoin feedsJoin = root.join(FeedEntry_.feedRelationships); - query.where(builder.equal(feedsJoin.get(FeedFeedEntry_.feed), feed)); - query.orderBy(builder.desc(feedsJoin.get(FeedFeedEntry_.entryUpdated))); - TypedQuery q = em.createQuery(query); - limit(q, offset, limit); - setTimeout(q, applicationSettingsService.get().getQueryTimeout()); - return q.getResultList(); + Predicate p1 = builder.equal(root.get(FeedEntry_.guidHash), + DigestUtils.sha1Hex(guid)); + Predicate p2 = builder.equal(root.get(FeedEntry_.url), url); + Predicate p3 = builder.equal(root.get(FeedEntry_.feed).get(Feed_.id), + feedId); + + query.where(p1, p2, p3); + + List list = em.createQuery(query).getResultList(); + return Iterables.getFirst(list, null); } public int delete(Date olderThan, int max) { @@ -90,20 +59,4 @@ public class FeedEntryDAO extends GenericDAO { delete(list); return deleted; } - - public int deleteWithoutFeeds(int max) { - CriteriaQuery query = builder.createQuery(getType()); - Root root = query.from(getType()); - - SetJoin join = root.join(FeedEntry_.feedRelationships, - JoinType.LEFT); - query.where(builder.isNull(join.get(FeedFeedEntry_.feed))); - TypedQuery q = em.createQuery(query); - q.setMaxResults(max); - - List list = q.getResultList(); - int deleted = list.size(); - delete(list); - return deleted; - } } diff --git a/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java b/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java index 1d9a0b0c..a8635d27 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java @@ -35,7 +35,6 @@ import com.commafeed.backend.model.FeedEntryContent_; import com.commafeed.backend.model.FeedEntryStatus; import com.commafeed.backend.model.FeedEntryStatus_; import com.commafeed.backend.model.FeedEntry_; -import com.commafeed.backend.model.FeedFeedEntry_; import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.Models; import com.commafeed.backend.model.User; @@ -52,7 +51,6 @@ public class FeedEntryStatusDAO extends GenericDAO { private static final String ALIAS_STATUS = "status"; private static final String ALIAS_ENTRY = "entry"; - private static final String ALIAS_FFE = "ffe"; private static final Comparator STATUS_COMPARATOR_DESC = new Comparator() { @Override @@ -94,8 +92,8 @@ public class FeedEntryStatusDAO extends GenericDAO { FeedSubscription sub, FeedEntry entry) { if (status == null) { Date unreadThreshold = getUnreadThreshold(); - boolean read = unreadThreshold == null ? false : entry - .getInserted().before(unreadThreshold); + boolean read = unreadThreshold == null ? false : entry.getUpdated() + .before(unreadThreshold); status = new FeedEntryStatus(sub.getUser(), sub, entry); status.setRead(read); status.setMarkable(!read); @@ -127,7 +125,12 @@ public class FeedEntryStatusDAO extends GenericDAO { TypedQuery q = em.createQuery(query); limit(q, offset, limit); setTimeout(q); - return lazyLoadContent(includeContent, q.getResultList()); + List statuses = q.getResultList(); + for (FeedEntryStatus status : statuses) { + status = handleStatus(status, status.getSubscription(), + status.getEntry()); + } + return lazyLoadContent(includeContent, statuses); } private Criteria buildSearchCriteria(FeedSubscription sub, @@ -136,16 +139,7 @@ public class FeedEntryStatusDAO extends GenericDAO { Criteria criteria = getSession().createCriteria(FeedEntry.class, ALIAS_ENTRY); - Criteria ffeJoin = criteria.createCriteria( - FeedEntry_.feedRelationships.getName(), ALIAS_FFE, - JoinType.INNER_JOIN); - ffeJoin.add(Restrictions.eq(FeedFeedEntry_.feed.getName(), - sub.getFeed())); - - if (newerThan != null) { - criteria.add(Restrictions.ge(FeedEntry_.inserted.getName(), - newerThan)); - } + criteria.add(Restrictions.eq(FeedEntry_.feed.getName(), sub.getFeed())); if (keywords != null) { Criteria contentJoin = criteria.createCriteria( @@ -175,29 +169,32 @@ public class FeedEntryStatusDAO extends GenericDAO { Date unreadThreshold = getUnreadThreshold(); if (unreadThreshold != null) { - criteria.add(Restrictions.ge(FeedEntry_.inserted.getName(), + criteria.add(Restrictions.ge(FeedEntry_.updated.getName(), unreadThreshold)); } } + if (newerThan != null) { + criteria.add(Restrictions.ge(FeedEntry_.inserted.getName(), + newerThan)); + } + if (last != null) { if (order == ReadingOrder.desc) { - ffeJoin.add(Restrictions.gt( - FeedFeedEntry_.entryUpdated.getName(), last)); + criteria.add(Restrictions.gt(FeedEntry_.updated.getName(), last)); } else { - ffeJoin.add(Restrictions.lt( - FeedFeedEntry_.entryUpdated.getName(), last)); + criteria.add(Restrictions.lt(FeedEntry_.updated.getName(), last)); } } if (order != null) { Order o = null; if (order == ReadingOrder.asc) { - o = Order.asc(FeedFeedEntry_.entryUpdated.getName()); + o = Order.asc(FeedEntry_.updated.getName()); } else { - o = Order.desc(FeedFeedEntry_.entryUpdated.getName()); + o = Order.desc(FeedEntry_.updated.getName()); } - ffeJoin.addOrder(o); + criteria.addOrder(o); } if (offset > -1) { criteria.setFirstResult(offset); @@ -256,7 +253,7 @@ public class FeedEntryStatusDAO extends GenericDAO { private Date getUnreadThreshold() { int keepStatusDays = applicationSettingsService.get() .getKeepStatusDays(); - return keepStatusDays > 0 ? DateUtils.addMinutes(new Date(), -1 + return keepStatusDays > 0 ? DateUtils.addDays(new Date(), -1 * keepStatusDays) : null; } diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java index e212e33a..2958aff0 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java @@ -1,7 +1,9 @@ package com.commafeed.backend.feeds; +import java.util.Arrays; import java.util.Collection; import java.util.Date; +import java.util.Iterator; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; @@ -11,6 +13,7 @@ import javax.annotation.PreDestroy; import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.time.DateUtils; import org.slf4j.Logger; @@ -72,7 +75,8 @@ public class FeedRefreshUpdater { public void init() { ApplicationSettings settings = applicationSettingsService.get(); int threads = Math.max(settings.getDatabaseUpdateThreads(), 1); - pool = new FeedRefreshExecutor("feed-refresh-updater", threads, 500 * threads); + pool = new FeedRefreshExecutor("feed-refresh-updater", threads, + 500 * threads); locks = Striped.lazyWeakLock(threads * 100000); } @@ -112,7 +116,7 @@ public class FeedRefreshUpdater { subscriptions = feedSubscriptionDAO .findByFeed(feed); } - ok &= updateEntry(feed, entry, subscriptions); + ok &= addEntry(feed, entry, subscriptions); metricsBean.entryCacheMiss(); } else { log.debug("cache hit for {}", entry.getUrl()); @@ -139,27 +143,42 @@ public class FeedRefreshUpdater { } } - private boolean updateEntry(final Feed feed, final FeedEntry entry, + private boolean addEntry(final Feed feed, final FeedEntry entry, final List subscriptions) { boolean success = false; - String key = StringUtils.trimToEmpty(entry.getGuid() + entry.getUrl()); - Lock lock = locks.get(key); - boolean locked = false; + // lock on feed, make sure we are not updating the same feed twice at + // the same time + String key1 = StringUtils.trimToEmpty("" + feed.getId()); + + // lock on content hash, make sure we are not updating the same entry + // twice at the same time + String key2 = DigestUtils.sha1Hex(entry.getContent().getContent()); + + Iterator iterator = locks.bulkGet(Arrays.asList(key1, key2)) + .iterator(); + Lock lock1 = iterator.next(); + Lock lock2 = iterator.next(); + boolean locked1 = false; + boolean locked2 = false; try { - locked = lock.tryLock(1, TimeUnit.MINUTES); - if (locked) { + locked1 = lock1.tryLock(1, TimeUnit.MINUTES); + locked2 = lock2.tryLock(1, TimeUnit.MINUTES); + if (locked1 && locked2) { feedUpdateService.updateEntry(feed, entry, subscriptions); success = true; } else { - log.error("lock timeout for " + feed.getUrl() + " - " + key); + log.error("lock timeout for " + feed.getUrl() + " - " + key1); } } catch (InterruptedException e) { log.error("interrupted while waiting for lock for " + feed.getUrl() + " : " + e.getMessage(), e); } finally { - if (locked) { - lock.unlock(); + if (locked1) { + lock1.unlock(); + } + if (locked2) { + lock2.unlock(); } } return success; diff --git a/src/main/java/com/commafeed/backend/model/Feed.java b/src/main/java/com/commafeed/backend/model/Feed.java index f29c3c8d..586cca23 100644 --- a/src/main/java/com/commafeed/backend/model/Feed.java +++ b/src/main/java/com/commafeed/backend/model/Feed.java @@ -4,6 +4,7 @@ import java.util.Date; import java.util.Set; import javax.persistence.Cacheable; +import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.OneToMany; @@ -107,8 +108,8 @@ public class Feed extends AbstractModel { @Column(length = 40) private String lastContentHash; - @OneToMany(mappedBy = "feed") - private Set entryRelationships; + @OneToMany(mappedBy = "feed", cascade = CascadeType.REMOVE) + private Set entries; @OneToMany(mappedBy = "feed") private Set subscriptions; @@ -325,12 +326,12 @@ public class Feed extends AbstractModel { this.normalizedUrlHash = normalizedUrlHash; } - public Set getEntryRelationships() { - return entryRelationships; + public Set getEntries() { + return entries; } - public void setEntryRelationships(Set entryRelationships) { - this.entryRelationships = entryRelationships; + public void setEntries(Set entries) { + this.entries = entries; } } diff --git a/src/main/java/com/commafeed/backend/model/FeedEntry.java b/src/main/java/com/commafeed/backend/model/FeedEntry.java index 486d42b1..1aeebb9c 100644 --- a/src/main/java/com/commafeed/backend/model/FeedEntry.java +++ b/src/main/java/com/commafeed/backend/model/FeedEntry.java @@ -9,6 +9,7 @@ import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.FetchType; import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; import javax.persistence.OneToMany; import javax.persistence.OneToOne; import javax.persistence.Table; @@ -32,8 +33,8 @@ public class FeedEntry extends AbstractModel { @Column(length = 40, nullable = false) private String guidHash; - @OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE) - private Set feedRelationships; + @ManyToOne(fetch = FetchType.LAZY) + private Feed feed; @OneToOne(cascade = CascadeType.ALL, fetch = FetchType.LAZY, optional = false) @JoinColumn(nullable = false, updatable = false) @@ -124,12 +125,12 @@ public class FeedEntry extends AbstractModel { this.author = author; } - public Set getFeedRelationships() { - return feedRelationships; + public Feed getFeed() { + return feed; } - public void setFeedRelationships(Set feedRelationships) { - this.feedRelationships = feedRelationships; + public void setFeed(Feed feed) { + this.feed = feed; } public FeedSubscription getSubscription() { diff --git a/src/main/java/com/commafeed/backend/model/FeedEntryContent.java b/src/main/java/com/commafeed/backend/model/FeedEntryContent.java index 7b4ccde8..fc6a545e 100644 --- a/src/main/java/com/commafeed/backend/model/FeedEntryContent.java +++ b/src/main/java/com/commafeed/backend/model/FeedEntryContent.java @@ -1,9 +1,12 @@ package com.commafeed.backend.model; +import java.util.Set; + import javax.persistence.Cacheable; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Lob; +import javax.persistence.OneToMany; import javax.persistence.Table; import org.hibernate.annotations.Cache; @@ -23,12 +26,21 @@ public class FeedEntryContent extends AbstractModel { @Column(length = Integer.MAX_VALUE) private String content; + @Column(length = 40) + private String contentHash; + + @Column(name = "author", length = 128) + private String author; + @Column(length = 2048) private String enclosureUrl; @Column(length = 255) private String enclosureType; + @OneToMany(mappedBy = "content") + private Set entries; + public String getContent() { return content; } @@ -61,4 +73,28 @@ public class FeedEntryContent extends AbstractModel { this.title = title; } + public String getContentHash() { + return contentHash; + } + + public void setContentHash(String contentHash) { + this.contentHash = contentHash; + } + + public String getAuthor() { + return author; + } + + public void setAuthor(String author) { + this.author = author; + } + + public Set getEntries() { + return entries; + } + + public void setEntries(Set entries) { + this.entries = entries; + } + } diff --git a/src/main/java/com/commafeed/backend/model/FeedFeedEntry.java b/src/main/java/com/commafeed/backend/model/FeedFeedEntry.java deleted file mode 100644 index cd76d8f2..00000000 --- a/src/main/java/com/commafeed/backend/model/FeedFeedEntry.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.commafeed.backend.model; - -import java.io.Serializable; -import java.util.Date; - -import javax.persistence.Cacheable; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.Id; -import javax.persistence.JoinColumn; -import javax.persistence.ManyToOne; -import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; - -import org.hibernate.annotations.Cache; -import org.hibernate.annotations.CacheConcurrencyStrategy; - -@Entity -@Table(name = "FEED_FEEDENTRIES") -@SuppressWarnings("serial") -@Cacheable -@Cache(usage = CacheConcurrencyStrategy.TRANSACTIONAL) -public class FeedFeedEntry implements Serializable { - - @Id - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "FEED_ID") - private Feed feed; - - @Id - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "FEEDENTRY_ID") - private FeedEntry entry; - - @Temporal(TemporalType.TIMESTAMP) - private Date entryUpdated; - - public FeedFeedEntry() { - - } - - public FeedFeedEntry(Feed feed, FeedEntry entry) { - this.feed = feed; - this.entry = entry; - this.entryUpdated = entry.getUpdated(); - } - - public Feed getFeed() { - return feed; - } - - public void setFeed(Feed feed) { - this.feed = feed; - } - - public FeedEntry getEntry() { - return entry; - } - - public void setEntry(FeedEntry entry) { - this.entry = entry; - } - - public Date getEntryUpdated() { - return entryUpdated; - } - - public void setEntryUpdated(Date entryUpdated) { - this.entryUpdated = entryUpdated; - } - -} diff --git a/src/main/java/com/commafeed/backend/services/FeedEntryContentService.java b/src/main/java/com/commafeed/backend/services/FeedEntryContentService.java new file mode 100644 index 00000000..da371af5 --- /dev/null +++ b/src/main/java/com/commafeed/backend/services/FeedEntryContentService.java @@ -0,0 +1,38 @@ +package com.commafeed.backend.services; + +import javax.inject.Inject; + +import org.apache.commons.codec.digest.DigestUtils; + +import com.commafeed.backend.dao.FeedEntryContentDAO; +import com.commafeed.backend.feeds.FeedUtils; +import com.commafeed.backend.model.FeedEntryContent; + +public class FeedEntryContentService { + + @Inject + FeedEntryContentDAO feedEntryContentDAO; + + /** + * this is NOT thread-safe + */ + public FeedEntryContent findOrCreate(FeedEntryContent content, + String baseUrl) { + + FeedEntryContent existing = feedEntryContentDAO.findExisting(content); + if (existing == null) { + content.setAuthor(FeedUtils.truncate( + FeedUtils.handleContent(content.getAuthor(), baseUrl, true), + 128)); + content.setTitle(FeedUtils.truncate( + FeedUtils.handleContent(content.getTitle(), baseUrl, true), + 2048)); + content.setContentHash(DigestUtils.sha1Hex(content.getContent())); + content.setContent(FeedUtils.handleContent(content.getContent(), + baseUrl, false)); + existing = content; + feedEntryContentDAO.saveOrUpdate(existing); + } + return existing; + } +} diff --git a/src/main/java/com/commafeed/backend/services/FeedEntryService.java b/src/main/java/com/commafeed/backend/services/FeedEntryService.java index 2ac1c31c..87560fef 100644 --- a/src/main/java/com/commafeed/backend/services/FeedEntryService.java +++ b/src/main/java/com/commafeed/backend/services/FeedEntryService.java @@ -58,10 +58,7 @@ public class FeedEntryService { } FeedEntryStatus status = feedEntryStatusDAO.getStatus(sub, entry); - if (status.isMarkable()) { - status.setStarred(starred); - feedEntryStatusDAO.saveOrUpdate(status); - } - + status.setStarred(starred); + feedEntryStatusDAO.saveOrUpdate(status); } } diff --git a/src/main/java/com/commafeed/backend/services/FeedSubscriptionService.java b/src/main/java/com/commafeed/backend/services/FeedSubscriptionService.java index 696de8f4..33d24f48 100644 --- a/src/main/java/com/commafeed/backend/services/FeedSubscriptionService.java +++ b/src/main/java/com/commafeed/backend/services/FeedSubscriptionService.java @@ -18,12 +18,9 @@ import com.commafeed.backend.feeds.FeedRefreshTaskGiver; import com.commafeed.backend.feeds.FeedUtils; import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedCategory; -import com.commafeed.backend.model.FeedEntry; -import com.commafeed.backend.model.FeedEntryStatus; import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.Models; import com.commafeed.backend.model.User; -import com.google.api.client.util.Lists; import com.google.api.client.util.Maps; public class FeedSubscriptionService { @@ -76,37 +73,16 @@ public class FeedSubscriptionService { Feed feed = feedService.findOrCreate(url); FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed); - boolean newSubscription = false; if (sub == null) { sub = new FeedSubscription(); sub.setFeed(feed); sub.setUser(user); - newSubscription = true; } sub.setCategory(category); sub.setPosition(0); sub.setTitle(FeedUtils.truncate(title, 128)); feedSubscriptionDAO.saveOrUpdate(sub); - if (newSubscription) { - try { - List statuses = Lists.newArrayList(); - List allEntries = feedEntryDAO.findByFeed(feed, 0, - 10); - for (FeedEntry entry : allEntries) { - FeedEntryStatus status = new FeedEntryStatus(user, sub, - entry); - status.setRead(false); - status.setSubscription(sub); - statuses.add(status); - } - feedEntryStatusDAO.saveOrUpdate(statuses); - } catch (Exception e) { - log.error( - "could not fetch initial statuses when importing {} : {}", - feed.getUrl(), e.getMessage()); - } - } taskGiver.add(feed); return feed; } diff --git a/src/main/java/com/commafeed/backend/services/FeedUpdateService.java b/src/main/java/com/commafeed/backend/services/FeedUpdateService.java index b683e823..bce11260 100644 --- a/src/main/java/com/commafeed/backend/services/FeedUpdateService.java +++ b/src/main/java/com/commafeed/backend/services/FeedUpdateService.java @@ -8,17 +8,16 @@ import javax.inject.Inject; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; +import org.apache.commons.codec.digest.DigestUtils; + import com.commafeed.backend.MetricsBean; import com.commafeed.backend.cache.CacheService; import com.commafeed.backend.dao.FeedEntryDAO; -import com.commafeed.backend.dao.FeedEntryDAO.EntryWithFeed; import com.commafeed.backend.dao.FeedEntryStatusDAO; import com.commafeed.backend.dao.FeedSubscriptionDAO; -import com.commafeed.backend.feeds.FeedUtils; import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntryContent; -import com.commafeed.backend.model.FeedFeedEntry; import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.User; import com.google.common.collect.Lists; @@ -43,41 +42,35 @@ public class FeedUpdateService { @Inject CacheService cache; + + @Inject + FeedEntryContentService feedEntryContentService; + /** + * this is NOT thread-safe + */ public void updateEntry(Feed feed, FeedEntry entry, List subscriptions) { - EntryWithFeed existing = feedEntryDAO.findExisting(entry.getGuid(), + FeedEntry existing = feedEntryDAO.findExisting(entry.getGuid(), entry.getUrl(), feed.getId()); - - FeedEntry update = null; - FeedFeedEntry ffe = null; - if (existing == null) { - entry.setAuthor(FeedUtils.truncate(FeedUtils.handleContent( - entry.getAuthor(), feed.getLink(), true), 128)); - FeedEntryContent content = entry.getContent(); - content.setTitle(FeedUtils.truncate(FeedUtils.handleContent( - content.getTitle(), feed.getLink(), true), 2048)); - content.setContent(FeedUtils.handleContent(content.getContent(), - feed.getLink(), false)); - - entry.setInserted(new Date()); - ffe = new FeedFeedEntry(feed, entry); - - update = entry; - } else if (existing.ffe == null) { - ffe = new FeedFeedEntry(feed, existing.entry); - update = existing.entry; + if (existing != null) { + return; } - if (update != null) { - List users = Lists.newArrayList(); - for (FeedSubscription sub : subscriptions) { - users.add(sub.getUser()); - } - cache.invalidateUserData(users.toArray(new User[0])); - feedEntryDAO.saveOrUpdate(update); - em.persist(ffe); + FeedEntryContent content = feedEntryContentService.findOrCreate( + entry.getContent(), feed.getLink()); + entry.setGuidHash(DigestUtils.sha1Hex(entry.getGuid())); + entry.setContent(content); + entry.setInserted(new Date()); + entry.setFeed(feed); + + List users = Lists.newArrayList(); + for (FeedSubscription sub : subscriptions) { + User user = sub.getUser(); + users.add(user); } + feedEntryDAO.saveOrUpdate(entry); + cache.invalidateUserData(users.toArray(new User[0])); } } diff --git a/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java b/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java index 3761cd74..b0dcdfe7 100644 --- a/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java +++ b/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java @@ -256,7 +256,16 @@ public class AdminREST extends AbstractResourceREST { Map map = Maps.newHashMap(); map.put("feeds_without_subscriptions", cleaner.cleanFeedsWithoutSubscriptions()); - map.put("entries_without_feeds", cleaner.cleanEntriesWithoutFeeds()); + return Response.ok(map).build(); + } + + @Path("/cleanup/content") + @GET + @ApiOperation(value = "Content cleanup", notes = "Delete contents without entries") + public Response cleanupContents() { + Map map = Maps.newHashMap(); + map.put("contents_without_entries", + cleaner.cleanContentsWithoutEntries()); return Response.ok(map).build(); } diff --git a/src/main/resources/META-INF/orm.xml b/src/main/resources/META-INF/orm.xml index aa541a9c..46f570e0 100644 --- a/src/main/resources/META-INF/orm.xml +++ b/src/main/resources/META-INF/orm.xml @@ -9,8 +9,4 @@ delete from FeedEntryStatus s where s.entryInserted < :date and s.starred = false - - select new com.commafeed.backend.dao.FeedEntryDAO$EntryWithFeed(e, f) FROM FeedEntry e LEFT JOIN e.feedRelationships f WITH f.feed.id = :feedId where e.guidHash = :guidHash and e.url = :url - - \ No newline at end of file diff --git a/src/main/resources/changelogs/db.changelog-1.2.xml b/src/main/resources/changelogs/db.changelog-1.2.xml new file mode 100644 index 00000000..155e685c --- /dev/null +++ b/src/main/resources/changelogs/db.changelog-1.2.xml @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/changelogs/db.changelog-master.xml b/src/main/resources/changelogs/db.changelog-master.xml index 498aa813..3fb6e8e1 100644 --- a/src/main/resources/changelogs/db.changelog-master.xml +++ b/src/main/resources/changelogs/db.changelog-master.xml @@ -5,5 +5,6 @@ - + + \ No newline at end of file diff --git a/src/main/webapp/js/controllers.js b/src/main/webapp/js/controllers.js index 3c0170b5..56ed4643 100644 --- a/src/main/webapp/js/controllers.js +++ b/src/main/webapp/js/controllers.js @@ -46,7 +46,7 @@ function($scope, FeedService, CategoryService, MobileService) { $scope.open = function() { $scope.sub = { - categoryId: $scope.sub.categoryId + categoryId: $scope.sub.categoryId || 'all' }; $scope.isOpen = true; };