store entry content externally and load only when needed

pom.xml

@@ -15,7 +15,7 @@
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<jpa.datasource.name>java:openejb/Resource/My DataSource</jpa.datasource.name>
-		<jpa.show_sql>false</jpa.show_sql>
+		<jpa.show_sql>true</jpa.show_sql>
 	</properties>
 
 	<build>

src/main/java/com/commafeed/backend/dao/FeedEntryStatusService.java

@@ -18,6 +18,7 @@ import org.apache.commons.lang.StringUtils;
 
 import com.commafeed.backend.model.Feed;
 import com.commafeed.backend.model.FeedCategory;
+import com.commafeed.backend.model.FeedEntryContent_;
 import com.commafeed.backend.model.FeedEntryStatus;
 import com.commafeed.backend.model.FeedEntryStatus_;
 import com.commafeed.backend.model.FeedEntry_;
@@ -58,12 +59,7 @@ public class FeedEntryStatusService extends GenericDAO<FeedEntryStatus> {
 	}
 
 	public List<FeedEntryStatus> getStatusesByKeywords(User user,
-			String keywords) {
+			String keywords, int offset, int limit, boolean includeContent) {
-		return getStatusesByKeywords(user, keywords, -1, -1);
-	}
-
-	public List<FeedEntryStatus> getStatusesByKeywords(User user,
-			String keywords, int offset, int limit) {
 
 		String joinedKeywords = StringUtils.join(
 				keywords.toLowerCase().split(" "), "%");
@@ -76,13 +72,17 @@ public class FeedEntryStatusService extends GenericDAO<FeedEntryStatus> {
 		predicates.add(builder.equal(root.get(FeedEntryStatus_.subscription)
 				.get(FeedSubscription_.user), user));
 
-		Predicate content = builder.like(
+		Predicate content = builder.like(builder.lower(root
-				builder.lower(root.get(FeedEntryStatus_.entry).get(
+				.get(FeedEntryStatus_.entry).get(FeedEntry_.content)
-						FeedEntry_.content)), joinedKeywords);
+				.get(FeedEntryContent_.content)), joinedKeywords);
 		Predicate title = builder.like(
-				builder.lower(root.get(FeedEntryStatus_.entry).get(
+				builder.lower(root.get(FeedEntryStatus_.entry)
-						FeedEntry_.title)), joinedKeywords);
+						.get(FeedEntry_.content).get(FeedEntryContent_.title)),
+				joinedKeywords);
 		predicates.add(builder.or(content, title));
+		if (includeContent) {
+			root.fetch(FeedEntryStatus_.entry).fetch(FeedEntry_.content);
+		}
 
 		query.where(predicates.toArray(new Predicate[0]));
 
@@ -94,12 +94,12 @@ public class FeedEntryStatusService extends GenericDAO<FeedEntryStatus> {
 	}
 
 	public List<FeedEntryStatus> getStatuses(User user, boolean unreadOnly,
-			ReadingOrder order) {
+			ReadingOrder order, boolean includeContent) {
-		return getStatuses(user, unreadOnly, -1, -1, order);
+		return getStatuses(user, unreadOnly, -1, -1, order, includeContent);
 	}
 
 	public List<FeedEntryStatus> getStatuses(User user, boolean unreadOnly,
-			int offset, int limit, ReadingOrder order) {
+			int offset, int limit, ReadingOrder order, boolean includeContent) {
 		CriteriaQuery<FeedEntryStatus> query = builder.createQuery(getType());
 		Root<FeedEntryStatus> root = query.from(getType());
 
@@ -109,6 +109,11 @@ public class FeedEntryStatusService extends GenericDAO<FeedEntryStatus> {
 		if (unreadOnly) {
 			predicates.add(builder.isFalse(root.get(FeedEntryStatus_.read)));
 		}
+
+		if (includeContent) {
+			root.fetch(FeedEntryStatus_.entry).fetch(FeedEntry_.content);
+		}
+
 		query.where(predicates.toArray(new Predicate[0]));
 		orderBy(query, root, order);
 
@@ -134,12 +139,14 @@ public class FeedEntryStatusService extends GenericDAO<FeedEntryStatus> {
 	}
 
 	public List<FeedEntryStatus> getStatuses(Feed feed, User user,
-			boolean unreadOnly, ReadingOrder order) {
+			boolean unreadOnly, ReadingOrder order, boolean includeContent) {
-		return getStatuses(feed, user, unreadOnly, -1, -1, order);
+		return getStatuses(feed, user, unreadOnly, -1, -1, order,
+				includeContent);
 	}
 
 	public List<FeedEntryStatus> getStatuses(Feed feed, User user,
-			boolean unreadOnly, int offset, int limit, ReadingOrder order) {
+			boolean unreadOnly, int offset, int limit, ReadingOrder order,
+			boolean includeContent) {
 
 		CriteriaQuery<FeedEntryStatus> query = builder.createQuery(getType());
 		Root<FeedEntryStatus> root = query.from(getType());
@@ -152,6 +159,11 @@ public class FeedEntryStatusService extends GenericDAO<FeedEntryStatus> {
 		if (unreadOnly) {
 			predicates.add(builder.isFalse(root.get(FeedEntryStatus_.read)));
 		}
+
+		if (includeContent) {
+			root.fetch(FeedEntryStatus_.entry).fetch(FeedEntry_.content);
+		}
+
 		query.where(predicates.toArray(new Predicate[0]));
 
 		orderBy(query, root, order);
@@ -162,13 +174,15 @@ public class FeedEntryStatusService extends GenericDAO<FeedEntryStatus> {
 	}
 
 	public List<FeedEntryStatus> getStatuses(List<FeedCategory> categories,
-			User user, boolean unreadOnly, ReadingOrder order) {
+			User user, boolean unreadOnly, ReadingOrder order,
-		return getStatuses(categories, user, unreadOnly, -1, -1, order);
+			boolean includeContent) {
+		return getStatuses(categories, user, unreadOnly, -1, -1, order,
+				includeContent);
 	}
 
 	public List<FeedEntryStatus> getStatuses(List<FeedCategory> categories,
 			User user, boolean unreadOnly, int offset, int limit,
-			ReadingOrder order) {
+			ReadingOrder order, boolean includeContent) {
 
 		CriteriaQuery<FeedEntryStatus> query = builder.createQuery(getType());
 		Root<FeedEntryStatus> root = query.from(getType());
@@ -181,6 +195,11 @@ public class FeedEntryStatusService extends GenericDAO<FeedEntryStatus> {
 		if (unreadOnly) {
 			predicates.add(builder.isFalse(root.get(FeedEntryStatus_.read)));
 		}
+
+		if (includeContent) {
+			root.fetch(FeedEntryStatus_.entry).fetch(FeedEntry_.content);
+		}
+
 		query.where(predicates.toArray(new Predicate[0]));
 
 		orderBy(query, root, order);
@@ -213,20 +232,20 @@ public class FeedEntryStatusService extends GenericDAO<FeedEntryStatus> {
 
 	public void markFeedEntries(User user, Feed feed, Date olderThan) {
 		List<FeedEntryStatus> statuses = getStatuses(feed, user, true,
-				ReadingOrder.desc);
+				ReadingOrder.desc, false);
 		update(markList(statuses, olderThan));
 	}
 
 	public void markCategoryEntries(User user, List<FeedCategory> categories,
 			Date olderThan) {
 		List<FeedEntryStatus> statuses = getStatuses(categories, user, true,
-				ReadingOrder.desc);
+				ReadingOrder.desc, false);
 		update(markList(statuses, olderThan));
 	}
 
 	public void markAllEntries(User user, Date olderThan) {
 		List<FeedEntryStatus> statuses = getStatuses(user, true,
-				ReadingOrder.desc);
+				ReadingOrder.desc, false);
 		update(markList(statuses, olderThan));
 	}
 

src/main/java/com/commafeed/backend/feeds/FeedParser.java

@@ -15,6 +15,7 @@ import org.xml.sax.InputSource;
 
 import com.commafeed.backend.model.Feed;
 import com.commafeed.backend.model.FeedEntry;
+import com.commafeed.backend.model.FeedEntryContent;
 import com.google.common.base.Function;
 import com.google.common.collect.Collections2;
 import com.google.common.collect.Iterables;
@@ -48,17 +49,19 @@ public class FeedParser {
 			for (SyndEntry item : items) {
 				FeedEntry entry = new FeedEntry();
 				entry.setGuid(item.getUri());
-				entry.setTitle(handleContent(item.getTitle()));
-				entry.setContent(handleContent(getContent(item)));
 				entry.setUrl(item.getLink());
 				entry.setUpdated(getUpdateDate(item));
 
+				FeedEntryContent content = new FeedEntryContent();
+				content.setContent(handleContent(getContent(item)));
+				content.setTitle(handleContent(item.getTitle()));
 				SyndEnclosure enclosure = (SyndEnclosure) Iterables.getFirst(
 						item.getEnclosures(), null);
 				if (enclosure != null) {
-					entry.setEnclosureUrl(enclosure.getUrl());
+					content.setEnclosureUrl(enclosure.getUrl());
-					entry.setEnclosureType(enclosure.getType());
+					content.setEnclosureType(enclosure.getType());
 				}
+				entry.setContent(content);
 
 				feed.getEntries().add(entry);
 			}
@@ -103,7 +106,6 @@ public class FeedParser {
 			Whitelist whitelist = Whitelist.relaxed();
 			whitelist.addEnforcedAttribute("a", "target", "_blank");
 
-			// TODO evaluate potential security issues
 			whitelist.addTags("iframe");
 			whitelist.addAttributes("iframe", "src", "height", "width",
 					"allowfullscreen", "frameborder");

src/main/java/com/commafeed/backend/model/FeedEntry.java

@@ -3,13 +3,15 @@ package com.commafeed.backend.model;
 import java.util.Date;
 import java.util.Set;
 
+import javax.persistence.CascadeType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
+import javax.persistence.FetchType;
 import javax.persistence.JoinColumn;
 import javax.persistence.JoinTable;
-import javax.persistence.Lob;
 import javax.persistence.ManyToMany;
 import javax.persistence.OneToMany;
+import javax.persistence.OneToOne;
 import javax.persistence.Table;
 import javax.persistence.Temporal;
 import javax.persistence.TemporalType;
@@ -30,18 +32,9 @@ public class FeedEntry extends AbstractModel {
 	@JoinTable(name = "FEED_FEEDENTRIES", joinColumns = { @JoinColumn(name = "FEED_ID", nullable = false, updatable = false) }, inverseJoinColumns = { @JoinColumn(name = "FEEDENTRY_ID", nullable = false, updatable = false) })
 	private Set<Feed> feeds = Sets.newHashSet();
 
-	@Column(length = 2048)
+	@OneToOne(cascade = CascadeType.ALL, fetch = FetchType.LAZY, optional = false)
-	private String title;
+	@JoinColumn(nullable = false, updatable = false)
-
+	private FeedEntryContent content;
-	@Lob
-	@Column(length = Integer.MAX_VALUE)
-	private String content;
-
-	@Column(length = 2048)
-	private String enclosureUrl;
-
-	@Column(length = 255)
-	private String enclosureType;
 
 	@Column(length = 2048)
 	private String url;
@@ -65,22 +58,6 @@ public class FeedEntry extends AbstractModel {
 		this.guid = guid;
 	}
 
-	public String getTitle() {
-		return title;
-	}
-
-	public void setTitle(String title) {
-		this.title = title;
-	}
-
-	public String getContent() {
-		return content;
-	}
-
-	public void setContent(String content) {
-		this.content = content;
-	}
-
 	public String getUrl() {
 		return url;
 	}
@@ -121,20 +98,12 @@ public class FeedEntry extends AbstractModel {
 		this.inserted = inserted;
 	}
 
-	public String getEnclosureUrl() {
+	public FeedEntryContent getContent() {
-		return enclosureUrl;
+		return content;
 	}
 
-	public void setEnclosureUrl(String enclosureUrl) {
+	public void setContent(FeedEntryContent content) {
-		this.enclosureUrl = enclosureUrl;
+		this.content = content;
-	}
-
-	public String getEnclosureType() {
-		return enclosureType;
-	}
-
-	public void setEnclosureType(String enclosureType) {
-		this.enclosureType = enclosureType;
 	}
 
 }

src/main/java/com/commafeed/backend/model/FeedEntryContent.java

@@ -0,0 +1,58 @@
+package com.commafeed.backend.model;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Lob;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "FEEDENTRYCONTENTS")
+@SuppressWarnings("serial")
+public class FeedEntryContent extends AbstractModel {
+
+	@Column(length = 2048)
+	private String title;
+
+	@Lob
+	@Column(length = Integer.MAX_VALUE)
+	private String content;
+
+	@Column(length = 2048)
+	private String enclosureUrl;
+
+	@Column(length = 255)
+	private String enclosureType;
+
+	public String getContent() {
+		return content;
+	}
+
+	public void setContent(String content) {
+		this.content = content;
+	}
+
+	public String getEnclosureUrl() {
+		return enclosureUrl;
+	}
+
+	public void setEnclosureUrl(String enclosureUrl) {
+		this.enclosureUrl = enclosureUrl;
+	}
+
+	public String getEnclosureType() {
+		return enclosureType;
+	}
+
+	public void setEnclosureType(String enclosureType) {
+		this.enclosureType = enclosureType;
+	}
+
+	public String getTitle() {
+		return title;
+	}
+
+	public void setTitle(String title) {
+		this.title = title;
+	}
+
+}

src/main/java/com/commafeed/frontend/rest/resources/AdminUsersREST.java

@@ -136,7 +136,7 @@ public class AdminUsersREST extends AbstractREST {
 					.entity("You cannot delete the admin user.").build();
 		}
 		feedEntryStatusService.delete(feedEntryStatusService.getStatuses(user,
-				false, ReadingOrder.desc));
+				false, ReadingOrder.desc, false));
 		feedSubscriptionService.delete(feedSubscriptionService.findAll(user));
 		feedCategoryService.delete(feedCategoryService.findAll(user));
 		userSettingsService.delete(userSettingsService.findByUser(user));

src/main/java/com/commafeed/frontend/rest/resources/EntriesREST.java

@@ -63,7 +63,7 @@ public class EntriesREST extends AbstractREST {
 
 				List<FeedEntryStatus> unreadEntries = feedEntryStatusService
 						.getStatuses(subscription.getFeed(), getUser(),
-								unreadOnly, offset, limit, order);
+								unreadOnly, offset, limit, order, true);
 				for (FeedEntryStatus status : unreadEntries) {
 					entries.getEntries().add(buildEntry(status));
 				}
@@ -75,7 +75,7 @@ public class EntriesREST extends AbstractREST {
 				entries.setName("All");
 				List<FeedEntryStatus> unreadEntries = feedEntryStatusService
 						.getStatuses(getUser(), unreadOnly, offset, limit,
-								order);
+								order, true);
 				for (FeedEntryStatus status : unreadEntries) {
 					entries.getEntries().add(buildEntry(status));
 				}
@@ -88,7 +88,7 @@ public class EntriesREST extends AbstractREST {
 							.findAllChildrenCategories(getUser(), feedCategory);
 					List<FeedEntryStatus> unreadEntries = feedEntryStatusService
 							.getStatuses(childrenCategories, getUser(),
-									unreadOnly, offset, limit, order);
+									unreadOnly, offset, limit, order, true);
 					for (FeedEntryStatus status : unreadEntries) {
 						entries.getEntries().add(buildEntry(status));
 					}
@@ -106,10 +106,11 @@ public class EntriesREST extends AbstractREST {
 
 		FeedEntry feedEntry = status.getEntry();
 		entry.setId(String.valueOf(status.getId()));
-		entry.setTitle(feedEntry.getTitle());
+		entry.setTitle(feedEntry.getContent().getTitle());
-		entry.setContent(feedEntry.getContent());
+		entry.setContent(feedEntry.getContent().getContent());
-		entry.setEnclosureUrl(status.getEntry().getEnclosureUrl());
+		entry.setEnclosureUrl(status.getEntry().getContent().getEnclosureUrl());
-		entry.setEnclosureType(status.getEntry().getEnclosureType());
+		entry.setEnclosureType(status.getEntry().getContent()
+				.getEnclosureType());
 		entry.setDate(feedEntry.getUpdated());
 		entry.setUrl(feedEntry.getUrl());
 
@@ -171,14 +172,16 @@ public class EntriesREST extends AbstractREST {
 
 	@Path("search")
 	@GET
-	public Entries searchEntries(@QueryParam("keywords") String keywords) {
+	public Entries searchEntries(@QueryParam("keywords") String keywords,
+			@DefaultValue("0") @QueryParam("offset") int offset,
+			@DefaultValue("-1") @QueryParam("limit") int limit) {
 		Preconditions.checkArgument(StringUtils.length(keywords) >= 3);
 
 		Entries entries = new Entries();
 
 		List<Entry> list = Lists.newArrayList();
 		List<FeedEntryStatus> entriesStatus = feedEntryStatusService
-				.getStatusesByKeywords(getUser(), keywords);
+				.getStatusesByKeywords(getUser(), keywords, offset, limit, true);
 		for (FeedEntryStatus status : entriesStatus) {
 			list.add(buildEntry(status));
 		}