remove many to many relationship between entries and feeds

src/main/java/com/commafeed/backend/DatabaseCleaner.java

@@ -10,6 +10,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.commafeed.backend.dao.FeedDAO;
+import com.commafeed.backend.dao.FeedEntryContentDAO;
 import com.commafeed.backend.dao.FeedEntryDAO;
 import com.commafeed.backend.dao.FeedSubscriptionDAO;
 import com.commafeed.backend.model.Feed;
@@ -28,6 +29,9 @@ public class DatabaseCleaner {
 
 	@Inject
 	FeedSubscriptionDAO feedSubscriptionDAO;
+	
+	@Inject
+	FeedEntryContentDAO feedEntryContentDAO;
 
 	@Inject
 	ApplicationSettingsService applicationSettingsService;
@@ -45,16 +49,16 @@ public class DatabaseCleaner {
 		return total;
 	}
 
-	public long cleanEntriesWithoutFeeds() {
+	public long cleanContentsWithoutEntries() {
 
 		long total = 0;
 		int deleted = -1;
 		do {
-			deleted = feedEntryDAO.deleteWithoutFeeds(100);
+			deleted = feedEntryContentDAO.deleteWithoutEntries(10);
 			total += deleted;
-			log.info("removed {} entries without feeds", total);
+			log.info("removed {} feeds without subscriptions", total);
 		} while (deleted != 0);
-		log.info("cleanup done: {} entries without feeds deleted", total);
+		log.info("cleanup done: {} feeds without subscriptions deleted", total);
 		return total;
 	}
 

src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java

@@ -0,0 +1,66 @@
+package com.commafeed.backend.dao;
+
+import java.util.List;
+
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.CriteriaQuery;
+import javax.persistence.criteria.Join;
+import javax.persistence.criteria.JoinType;
+import javax.persistence.criteria.Predicate;
+import javax.persistence.criteria.Root;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+import com.commafeed.backend.model.FeedEntry;
+import com.commafeed.backend.model.FeedEntryContent;
+import com.commafeed.backend.model.FeedEntryContent_;
+import com.commafeed.backend.model.FeedEntry_;
+import com.google.common.collect.Iterables;
+
+public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
+
+	public FeedEntryContent findExisting(FeedEntryContent content) {
+
+		CriteriaQuery<FeedEntryContent> query = builder.createQuery(getType());
+		Root<FeedEntryContent> root = query.from(getType());
+
+		Predicate p1 = builder.equal(root.get(FeedEntryContent_.contentHash),
+				DigestUtils.sha1Hex(content.getContent()));
+		Predicate p2 = null;
+		if (content.getTitle() == null) {
+			p2 = builder.isNull(root.get(FeedEntryContent_.title));
+		} else {
+			p2 = builder.equal(root.get(FeedEntryContent_.title),
+					content.getTitle());
+		}
+
+		Predicate p3 = null;
+		if (content.getAuthor() == null) {
+			p3 = builder.isNull(root.get(FeedEntryContent_.author));
+		} else {
+			p3 = builder.equal(root.get(FeedEntryContent_.author),
+					content.getAuthor());
+		}
+
+		query.where(p1, p2, p3);
+		TypedQuery<FeedEntryContent> q = em.createQuery(query);
+		return Iterables.getFirst(q.getResultList(), null);
+
+	}
+
+	public int deleteWithoutEntries(int max) {
+		CriteriaQuery<FeedEntryContent> query = builder.createQuery(getType());
+		Root<FeedEntryContent> root = query.from(getType());
+
+		Join<FeedEntryContent, FeedEntry> join = root.join(
+				FeedEntryContent_.entries, JoinType.LEFT);
+		query.where(builder.isNull(join.get(FeedEntry_.id)));
+		TypedQuery<FeedEntryContent> q = em.createQuery(query);
+		q.setMaxResults(max);
+
+		List<FeedEntryContent> list = q.getResultList();
+		int deleted = list.size();
+		return deleted;
+
+	}
+}

src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java

@@ -7,19 +7,16 @@ import javax.ejb.Stateless;
 import javax.inject.Inject;
 import javax.persistence.TypedQuery;
 import javax.persistence.criteria.CriteriaQuery;
-import javax.persistence.criteria.JoinType;
+import javax.persistence.criteria.Predicate;
 import javax.persistence.criteria.Root;
-import javax.persistence.criteria.SetJoin;
 
 import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.commafeed.backend.model.Feed;
 import com.commafeed.backend.model.FeedEntry;
 import com.commafeed.backend.model.FeedEntry_;
-import com.commafeed.backend.model.FeedFeedEntry;
-import com.commafeed.backend.model.FeedFeedEntry_;
+import com.commafeed.backend.model.Feed_;
 import com.commafeed.backend.services.ApplicationSettingsService;
 import com.google.common.collect.Iterables;
 
@@ -32,49 +29,21 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
 	protected static final Logger log = LoggerFactory
 			.getLogger(FeedEntryDAO.class);
 
-	public static class EntryWithFeed {
-		public FeedEntry entry;
-		public FeedFeedEntry ffe;
+	public FeedEntry findExisting(String guid, String url, Long feedId) {
 
-		public EntryWithFeed(FeedEntry entry, FeedFeedEntry ffe) {
-			this.entry = entry;
-			this.ffe = ffe;
-		}
-	}
-
-	public EntryWithFeed findExisting(String guid, String url, Long feedId) {
-
-		TypedQuery<EntryWithFeed> q = em.createNamedQuery(
-				"EntryStatus.existing", EntryWithFeed.class);
-		q.setParameter("guidHash", DigestUtils.sha1Hex(guid));
-		q.setParameter("url", url);
-		q.setParameter("feedId", feedId);
-
-		EntryWithFeed result = null;
-		List<EntryWithFeed> list = q.getResultList();
-		for (EntryWithFeed ewf : list) {
-			if (ewf.entry != null && ewf.ffe != null) {
-				result = ewf;
-				break;
-			}
-		}
-		if (result == null) {
-			result = Iterables.getFirst(list, null);
-		}
-		return result;
-	}
-
-	public List<FeedEntry> findByFeed(Feed feed, int offset, int limit) {
 		CriteriaQuery<FeedEntry> query = builder.createQuery(getType());
 		Root<FeedEntry> root = query.from(getType());
-		SetJoin<FeedEntry, FeedFeedEntry> feedsJoin = root.join(FeedEntry_.feedRelationships);
 
-		query.where(builder.equal(feedsJoin.get(FeedFeedEntry_.feed), feed));
-		query.orderBy(builder.desc(feedsJoin.get(FeedFeedEntry_.entryUpdated)));
-		TypedQuery<FeedEntry> q = em.createQuery(query);
-		limit(q, offset, limit);
-		setTimeout(q, applicationSettingsService.get().getQueryTimeout());
-		return q.getResultList();
+		Predicate p1 = builder.equal(root.get(FeedEntry_.guidHash),
+				DigestUtils.sha1Hex(guid));
+		Predicate p2 = builder.equal(root.get(FeedEntry_.url), url);
+		Predicate p3 = builder.equal(root.get(FeedEntry_.feed).get(Feed_.id),
+				feedId);
+
+		query.where(p1, p2, p3);
+
+		List<FeedEntry> list = em.createQuery(query).getResultList();
+		return Iterables.getFirst(list, null);
 	}
 
 	public int delete(Date olderThan, int max) {
@@ -90,20 +59,4 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
 		delete(list);
 		return deleted;
 	}
-
-	public int deleteWithoutFeeds(int max) {
-		CriteriaQuery<FeedEntry> query = builder.createQuery(getType());
-		Root<FeedEntry> root = query.from(getType());
-
-		SetJoin<FeedEntry, FeedFeedEntry> join = root.join(FeedEntry_.feedRelationships,
-				JoinType.LEFT);
-		query.where(builder.isNull(join.get(FeedFeedEntry_.feed)));
-		TypedQuery<FeedEntry> q = em.createQuery(query);
-		q.setMaxResults(max);
-
-		List<FeedEntry> list = q.getResultList();
-		int deleted = list.size();
-		delete(list);
-		return deleted;
-	}
 }

src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java

@@ -35,7 +35,6 @@ import com.commafeed.backend.model.FeedEntryContent_;
 import com.commafeed.backend.model.FeedEntryStatus;
 import com.commafeed.backend.model.FeedEntryStatus_;
 import com.commafeed.backend.model.FeedEntry_;
-import com.commafeed.backend.model.FeedFeedEntry_;
 import com.commafeed.backend.model.FeedSubscription;
 import com.commafeed.backend.model.Models;
 import com.commafeed.backend.model.User;
@@ -52,7 +51,6 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 
 	private static final String ALIAS_STATUS = "status";
 	private static final String ALIAS_ENTRY = "entry";
-	private static final String ALIAS_FFE = "ffe";
 
 	private static final Comparator<FeedEntryStatus> STATUS_COMPARATOR_DESC = new Comparator<FeedEntryStatus>() {
 		@Override
@@ -94,8 +92,8 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 			FeedSubscription sub, FeedEntry entry) {
 		if (status == null) {
 			Date unreadThreshold = getUnreadThreshold();
-			boolean read = unreadThreshold == null ? false : entry
-					.getInserted().before(unreadThreshold);
+			boolean read = unreadThreshold == null ? false : entry.getUpdated()
+					.before(unreadThreshold);
 			status = new FeedEntryStatus(sub.getUser(), sub, entry);
 			status.setRead(read);
 			status.setMarkable(!read);
@@ -127,7 +125,12 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 		TypedQuery<FeedEntryStatus> q = em.createQuery(query);
 		limit(q, offset, limit);
 		setTimeout(q);
-		return lazyLoadContent(includeContent, q.getResultList());
+		List<FeedEntryStatus> statuses = q.getResultList();
+		for (FeedEntryStatus status : statuses) {
+			status = handleStatus(status, status.getSubscription(),
+					status.getEntry());
+		}
+		return lazyLoadContent(includeContent, statuses);
 	}
 
 	private Criteria buildSearchCriteria(FeedSubscription sub,
@@ -136,16 +139,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 		Criteria criteria = getSession().createCriteria(FeedEntry.class,
 				ALIAS_ENTRY);
 
-		Criteria ffeJoin = criteria.createCriteria(
-				FeedEntry_.feedRelationships.getName(), ALIAS_FFE,
-				JoinType.INNER_JOIN);
-		ffeJoin.add(Restrictions.eq(FeedFeedEntry_.feed.getName(),
-				sub.getFeed()));
-
-		if (newerThan != null) {
-			criteria.add(Restrictions.ge(FeedEntry_.inserted.getName(),
-					newerThan));
-		}
+		criteria.add(Restrictions.eq(FeedEntry_.feed.getName(), sub.getFeed()));
 
 		if (keywords != null) {
 			Criteria contentJoin = criteria.createCriteria(
@@ -175,29 +169,32 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 
 			Date unreadThreshold = getUnreadThreshold();
 			if (unreadThreshold != null) {
-				criteria.add(Restrictions.ge(FeedEntry_.inserted.getName(),
+				criteria.add(Restrictions.ge(FeedEntry_.updated.getName(),
 						unreadThreshold));
 			}
 		}
 
+		if (newerThan != null) {
+			criteria.add(Restrictions.ge(FeedEntry_.inserted.getName(),
+					newerThan));
+		}
+
 		if (last != null) {
 			if (order == ReadingOrder.desc) {
-				ffeJoin.add(Restrictions.gt(
-						FeedFeedEntry_.entryUpdated.getName(), last));
+				criteria.add(Restrictions.gt(FeedEntry_.updated.getName(), last));
 			} else {
-				ffeJoin.add(Restrictions.lt(
-						FeedFeedEntry_.entryUpdated.getName(), last));
+				criteria.add(Restrictions.lt(FeedEntry_.updated.getName(), last));
 			}
 		}
 
 		if (order != null) {
 			Order o = null;
 			if (order == ReadingOrder.asc) {
-				o = Order.asc(FeedFeedEntry_.entryUpdated.getName());
+				o = Order.asc(FeedEntry_.updated.getName());
 			} else {
-				o = Order.desc(FeedFeedEntry_.entryUpdated.getName());
+				o = Order.desc(FeedEntry_.updated.getName());
 			}
-			ffeJoin.addOrder(o);
+			criteria.addOrder(o);
 		}
 		if (offset > -1) {
 			criteria.setFirstResult(offset);
@@ -256,7 +253,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 	private Date getUnreadThreshold() {
 		int keepStatusDays = applicationSettingsService.get()
 				.getKeepStatusDays();
-		return keepStatusDays > 0 ? DateUtils.addMinutes(new Date(), -1
+		return keepStatusDays > 0 ? DateUtils.addDays(new Date(), -1
 				* keepStatusDays) : null;
 	}
 

src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java

@@ -1,7 +1,9 @@
 package com.commafeed.backend.feeds;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Date;
+import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
@@ -11,6 +13,7 @@ import javax.annotation.PreDestroy;
 import javax.enterprise.context.ApplicationScoped;
 import javax.inject.Inject;
 
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.time.DateUtils;
 import org.slf4j.Logger;
@@ -72,7 +75,8 @@ public class FeedRefreshUpdater {
 	public void init() {
 		ApplicationSettings settings = applicationSettingsService.get();
 		int threads = Math.max(settings.getDatabaseUpdateThreads(), 1);
-		pool = new FeedRefreshExecutor("feed-refresh-updater", threads, 500 * threads);
+		pool = new FeedRefreshExecutor("feed-refresh-updater", threads,
+				500 * threads);
 		locks = Striped.lazyWeakLock(threads * 100000);
 	}
 
@@ -112,7 +116,7 @@ public class FeedRefreshUpdater {
 							subscriptions = feedSubscriptionDAO
 									.findByFeed(feed);
 						}
-						ok &= updateEntry(feed, entry, subscriptions);
+						ok &= addEntry(feed, entry, subscriptions);
 						metricsBean.entryCacheMiss();
 					} else {
 						log.debug("cache hit for {}", entry.getUrl());
@@ -139,27 +143,42 @@ public class FeedRefreshUpdater {
 		}
 	}
 
-	private boolean updateEntry(final Feed feed, final FeedEntry entry,
+	private boolean addEntry(final Feed feed, final FeedEntry entry,
 			final List<FeedSubscription> subscriptions) {
 		boolean success = false;
 
-		String key = StringUtils.trimToEmpty(entry.getGuid() + entry.getUrl());
-		Lock lock = locks.get(key);
-		boolean locked = false;
+		// lock on feed, make sure we are not updating the same feed twice at
+		// the same time
+		String key1 = StringUtils.trimToEmpty("" + feed.getId());
+
+		// lock on content hash, make sure we are not updating the same entry
+		// twice at the same time
+		String key2 = DigestUtils.sha1Hex(entry.getContent().getContent());
+
+		Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2))
+				.iterator();
+		Lock lock1 = iterator.next();
+		Lock lock2 = iterator.next();
+		boolean locked1 = false;
+		boolean locked2 = false;
 		try {
-			locked = lock.tryLock(1, TimeUnit.MINUTES);
-			if (locked) {
+			locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
+			locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
+			if (locked1 && locked2) {
 				feedUpdateService.updateEntry(feed, entry, subscriptions);
 				success = true;
 			} else {
-				log.error("lock timeout for " + feed.getUrl() + " - " + key);
+				log.error("lock timeout for " + feed.getUrl() + " - " + key1);
 			}
 		} catch (InterruptedException e) {
 			log.error("interrupted while waiting for lock for " + feed.getUrl()
 					+ " : " + e.getMessage(), e);
 		} finally {
-			if (locked) {
-				lock.unlock();
+			if (locked1) {
+				lock1.unlock();
+			}
+			if (locked2) {
+				lock2.unlock();
 			}
 		}
 		return success;

src/main/java/com/commafeed/backend/model/Feed.java

@@ -4,6 +4,7 @@ import java.util.Date;
 import java.util.Set;
 
 import javax.persistence.Cacheable;
+import javax.persistence.CascadeType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.OneToMany;
@@ -107,8 +108,8 @@ public class Feed extends AbstractModel {
 	@Column(length = 40)
 	private String lastContentHash;
 
-	@OneToMany(mappedBy = "feed")
-	private Set<FeedFeedEntry> entryRelationships;
+	@OneToMany(mappedBy = "feed", cascade = CascadeType.REMOVE)
+	private Set<FeedEntry> entries;
 
 	@OneToMany(mappedBy = "feed")
 	private Set<FeedSubscription> subscriptions;
@@ -325,12 +326,12 @@ public class Feed extends AbstractModel {
 		this.normalizedUrlHash = normalizedUrlHash;
 	}
 
-	public Set<FeedFeedEntry> getEntryRelationships() {
-		return entryRelationships;
+	public Set<FeedEntry> getEntries() {
+		return entries;
 	}
 
-	public void setEntryRelationships(Set<FeedFeedEntry> entryRelationships) {
-		this.entryRelationships = entryRelationships;
+	public void setEntries(Set<FeedEntry> entries) {
+		this.entries = entries;
 	}
 
 }

src/main/java/com/commafeed/backend/model/FeedEntry.java

@@ -9,6 +9,7 @@ import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.FetchType;
 import javax.persistence.JoinColumn;
+import javax.persistence.ManyToOne;
 import javax.persistence.OneToMany;
 import javax.persistence.OneToOne;
 import javax.persistence.Table;
@@ -32,8 +33,8 @@ public class FeedEntry extends AbstractModel {
 	@Column(length = 40, nullable = false)
 	private String guidHash;
 
-	@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
-	private Set<FeedFeedEntry> feedRelationships;
+	@ManyToOne(fetch = FetchType.LAZY)
+	private Feed feed;
 
 	@OneToOne(cascade = CascadeType.ALL, fetch = FetchType.LAZY, optional = false)
 	@JoinColumn(nullable = false, updatable = false)
@@ -124,12 +125,12 @@ public class FeedEntry extends AbstractModel {
 		this.author = author;
 	}
 
-	public Set<FeedFeedEntry> getFeedRelationships() {
-		return feedRelationships;
+	public Feed getFeed() {
+		return feed;
 	}
 
-	public void setFeedRelationships(Set<FeedFeedEntry> feedRelationships) {
-		this.feedRelationships = feedRelationships;
+	public void setFeed(Feed feed) {
+		this.feed = feed;
 	}
 
 	public FeedSubscription getSubscription() {

src/main/java/com/commafeed/backend/model/FeedEntryContent.java

@@ -1,9 +1,12 @@
 package com.commafeed.backend.model;
 
+import java.util.Set;
+
 import javax.persistence.Cacheable;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Lob;
+import javax.persistence.OneToMany;
 import javax.persistence.Table;
 
 import org.hibernate.annotations.Cache;
@@ -23,12 +26,21 @@ public class FeedEntryContent extends AbstractModel {
 	@Column(length = Integer.MAX_VALUE)
 	private String content;
 
+	@Column(length = 40)
+	private String contentHash;
+
+	@Column(name = "author", length = 128)
+	private String author;
+
 	@Column(length = 2048)
 	private String enclosureUrl;
 
 	@Column(length = 255)
 	private String enclosureType;
 
+	@OneToMany(mappedBy = "content")
+	private Set<FeedEntry> entries;
+
 	public String getContent() {
 		return content;
 	}
@@ -61,4 +73,28 @@ public class FeedEntryContent extends AbstractModel {
 		this.title = title;
 	}
 
+	public String getContentHash() {
+		return contentHash;
+	}
+
+	public void setContentHash(String contentHash) {
+		this.contentHash = contentHash;
+	}
+
+	public String getAuthor() {
+		return author;
+	}
+
+	public void setAuthor(String author) {
+		this.author = author;
+	}
+
+	public Set<FeedEntry> getEntries() {
+		return entries;
+	}
+
+	public void setEntries(Set<FeedEntry> entries) {
+		this.entries = entries;
+	}
+
 }

src/main/java/com/commafeed/backend/model/FeedFeedEntry.java

@@ -1,73 +0,0 @@
-package com.commafeed.backend.model;
-
-import java.io.Serializable;
-import java.util.Date;
-
-import javax.persistence.Cacheable;
-import javax.persistence.Entity;
-import javax.persistence.FetchType;
-import javax.persistence.Id;
-import javax.persistence.JoinColumn;
-import javax.persistence.ManyToOne;
-import javax.persistence.Table;
-import javax.persistence.Temporal;
-import javax.persistence.TemporalType;
-
-import org.hibernate.annotations.Cache;
-import org.hibernate.annotations.CacheConcurrencyStrategy;
-
-@Entity
-@Table(name = "FEED_FEEDENTRIES")
-@SuppressWarnings("serial")
-@Cacheable
-@Cache(usage = CacheConcurrencyStrategy.TRANSACTIONAL)
-public class FeedFeedEntry implements Serializable {
-
-	@Id
-	@ManyToOne(fetch = FetchType.LAZY)
-	@JoinColumn(name = "FEED_ID")
-	private Feed feed;
-
-	@Id
-	@ManyToOne(fetch = FetchType.LAZY)
-	@JoinColumn(name = "FEEDENTRY_ID")
-	private FeedEntry entry;
-
-	@Temporal(TemporalType.TIMESTAMP)
-	private Date entryUpdated;
-
-	public FeedFeedEntry() {
-
-	}
-
-	public FeedFeedEntry(Feed feed, FeedEntry entry) {
-		this.feed = feed;
-		this.entry = entry;
-		this.entryUpdated = entry.getUpdated();
-	}
-
-	public Feed getFeed() {
-		return feed;
-	}
-
-	public void setFeed(Feed feed) {
-		this.feed = feed;
-	}
-
-	public FeedEntry getEntry() {
-		return entry;
-	}
-
-	public void setEntry(FeedEntry entry) {
-		this.entry = entry;
-	}
-
-	public Date getEntryUpdated() {
-		return entryUpdated;
-	}
-
-	public void setEntryUpdated(Date entryUpdated) {
-		this.entryUpdated = entryUpdated;
-	}
-
-}

src/main/java/com/commafeed/backend/services/FeedEntryContentService.java

@@ -0,0 +1,38 @@
+package com.commafeed.backend.services;
+
+import javax.inject.Inject;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+import com.commafeed.backend.dao.FeedEntryContentDAO;
+import com.commafeed.backend.feeds.FeedUtils;
+import com.commafeed.backend.model.FeedEntryContent;
+
+public class FeedEntryContentService {
+
+	@Inject
+	FeedEntryContentDAO feedEntryContentDAO;
+
+	/**
+	 * this is NOT thread-safe
+	 */
+	public FeedEntryContent findOrCreate(FeedEntryContent content,
+			String baseUrl) {
+
+		FeedEntryContent existing = feedEntryContentDAO.findExisting(content);
+		if (existing == null) {
+			content.setAuthor(FeedUtils.truncate(
+					FeedUtils.handleContent(content.getAuthor(), baseUrl, true),
+					128));
+			content.setTitle(FeedUtils.truncate(
+					FeedUtils.handleContent(content.getTitle(), baseUrl, true),
+					2048));
+			content.setContentHash(DigestUtils.sha1Hex(content.getContent()));
+			content.setContent(FeedUtils.handleContent(content.getContent(),
+					baseUrl, false));
+			existing = content;
+			feedEntryContentDAO.saveOrUpdate(existing);
+		}
+		return existing;
+	}
+}

src/main/java/com/commafeed/backend/services/FeedEntryService.java

@@ -58,10 +58,7 @@ public class FeedEntryService {
 		}
 
 		FeedEntryStatus status = feedEntryStatusDAO.getStatus(sub, entry);
-		if (status.isMarkable()) {
-			status.setStarred(starred);
-			feedEntryStatusDAO.saveOrUpdate(status);
-		}
-
+		status.setStarred(starred);
+		feedEntryStatusDAO.saveOrUpdate(status);
 	}
 }

src/main/java/com/commafeed/backend/services/FeedSubscriptionService.java

@@ -18,12 +18,9 @@ import com.commafeed.backend.feeds.FeedRefreshTaskGiver;
 import com.commafeed.backend.feeds.FeedUtils;
 import com.commafeed.backend.model.Feed;
 import com.commafeed.backend.model.FeedCategory;
-import com.commafeed.backend.model.FeedEntry;
-import com.commafeed.backend.model.FeedEntryStatus;
 import com.commafeed.backend.model.FeedSubscription;
 import com.commafeed.backend.model.Models;
 import com.commafeed.backend.model.User;
-import com.google.api.client.util.Lists;
 import com.google.api.client.util.Maps;
 
 public class FeedSubscriptionService {
@@ -76,37 +73,16 @@ public class FeedSubscriptionService {
 		Feed feed = feedService.findOrCreate(url);
 
 		FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
-		boolean newSubscription = false;
 		if (sub == null) {
 			sub = new FeedSubscription();
 			sub.setFeed(feed);
 			sub.setUser(user);
-			newSubscription = true;
 		}
 		sub.setCategory(category);
 		sub.setPosition(0);
 		sub.setTitle(FeedUtils.truncate(title, 128));
 		feedSubscriptionDAO.saveOrUpdate(sub);
 
-		if (newSubscription) {
-			try {
-				List<FeedEntryStatus> statuses = Lists.newArrayList();
-				List<FeedEntry> allEntries = feedEntryDAO.findByFeed(feed, 0,
-						10);
-				for (FeedEntry entry : allEntries) {
-					FeedEntryStatus status = new FeedEntryStatus(user, sub,
-							entry);
-					status.setRead(false);
-					status.setSubscription(sub);
-					statuses.add(status);
-				}
-				feedEntryStatusDAO.saveOrUpdate(statuses);
-			} catch (Exception e) {
-				log.error(
-						"could not fetch initial statuses when importing {} : {}",
-						feed.getUrl(), e.getMessage());
-			}
-		}
 		taskGiver.add(feed);
 		return feed;
 	}

src/main/java/com/commafeed/backend/services/FeedUpdateService.java

@@ -8,17 +8,16 @@ import javax.inject.Inject;
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
 
+import org.apache.commons.codec.digest.DigestUtils;
+
 import com.commafeed.backend.MetricsBean;
 import com.commafeed.backend.cache.CacheService;
 import com.commafeed.backend.dao.FeedEntryDAO;
-import com.commafeed.backend.dao.FeedEntryDAO.EntryWithFeed;
 import com.commafeed.backend.dao.FeedEntryStatusDAO;
 import com.commafeed.backend.dao.FeedSubscriptionDAO;
-import com.commafeed.backend.feeds.FeedUtils;
 import com.commafeed.backend.model.Feed;
 import com.commafeed.backend.model.FeedEntry;
 import com.commafeed.backend.model.FeedEntryContent;
-import com.commafeed.backend.model.FeedFeedEntry;
 import com.commafeed.backend.model.FeedSubscription;
 import com.commafeed.backend.model.User;
 import com.google.common.collect.Lists;
@@ -43,41 +42,35 @@ public class FeedUpdateService {
 
 	@Inject
 	CacheService cache;
+	
+	@Inject
+	FeedEntryContentService feedEntryContentService;
 
+	/**
+	 * this is NOT thread-safe
+	 */
 	public void updateEntry(Feed feed, FeedEntry entry,
 			List<FeedSubscription> subscriptions) {
 
-		EntryWithFeed existing = feedEntryDAO.findExisting(entry.getGuid(),
+		FeedEntry existing = feedEntryDAO.findExisting(entry.getGuid(),
 				entry.getUrl(), feed.getId());
-
-		FeedEntry update = null;
-		FeedFeedEntry ffe = null;
-		if (existing == null) {
-			entry.setAuthor(FeedUtils.truncate(FeedUtils.handleContent(
-					entry.getAuthor(), feed.getLink(), true), 128));
-			FeedEntryContent content = entry.getContent();
-			content.setTitle(FeedUtils.truncate(FeedUtils.handleContent(
-					content.getTitle(), feed.getLink(), true), 2048));
-			content.setContent(FeedUtils.handleContent(content.getContent(),
-					feed.getLink(), false));
-
-			entry.setInserted(new Date());
-			ffe = new FeedFeedEntry(feed, entry);
-
-			update = entry;
-		} else if (existing.ffe == null) {
-			ffe = new FeedFeedEntry(feed, existing.entry);
-			update = existing.entry;
+		if (existing != null) {
+			return;
 		}
 
-		if (update != null) {
-			List<User> users = Lists.newArrayList();
-			for (FeedSubscription sub : subscriptions) {
-				users.add(sub.getUser());
-			}
-			cache.invalidateUserData(users.toArray(new User[0]));
-			feedEntryDAO.saveOrUpdate(update);
-			em.persist(ffe);
+		FeedEntryContent content = feedEntryContentService.findOrCreate(
+				entry.getContent(), feed.getLink());
+		entry.setGuidHash(DigestUtils.sha1Hex(entry.getGuid()));
+		entry.setContent(content);
+		entry.setInserted(new Date());
+		entry.setFeed(feed);
+
+		List<User> users = Lists.newArrayList();
+		for (FeedSubscription sub : subscriptions) {
+			User user = sub.getUser();
+			users.add(user);
 		}
+		feedEntryDAO.saveOrUpdate(entry);
+		cache.invalidateUserData(users.toArray(new User[0]));
 	}
 }

src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java

@@ -256,7 +256,16 @@ public class AdminREST extends AbstractResourceREST {
 		Map<String, Long> map = Maps.newHashMap();
 		map.put("feeds_without_subscriptions",
 				cleaner.cleanFeedsWithoutSubscriptions());
-		map.put("entries_without_feeds", cleaner.cleanEntriesWithoutFeeds());
+		return Response.ok(map).build();
+	}
+	
+	@Path("/cleanup/content")
+	@GET
+	@ApiOperation(value = "Content cleanup", notes = "Delete contents without entries")
+	public Response cleanupContents() {
+		Map<String, Long> map = Maps.newHashMap();
+		map.put("contents_without_entries",
+				cleaner.cleanContentsWithoutEntries());
 		return Response.ok(map).build();
 	}