implement faster querying by fetching directly what we need

commafeed-server/src/main/java/com/commafeed/backend/FixedSizeSortedList.java

@@ -1,49 +0,0 @@
-package com.commafeed.backend;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-
-/**
- * List wrapper that sorts its elements in the order provided by given comparator and ensure a maximum capacity.
- * 
- * 
- */
-public class FixedSizeSortedList<E> {
-
-	private final List<E> inner;
-
-	private final Comparator<? super E> comparator;
-	private final int capacity;
-
-	public FixedSizeSortedList(int capacity, Comparator<? super E> comparator) {
-		this.inner = new ArrayList<>(Math.max(0, capacity));
-		this.capacity = capacity < 0 ? Integer.MAX_VALUE : capacity;
-		this.comparator = comparator;
-	}
-
-	public void add(E e) {
-		int position = Math.abs(Collections.binarySearch(inner, e, comparator) + 1);
-		if (isFull()) {
-			if (position < inner.size()) {
-				inner.remove(inner.size() - 1);
-				inner.add(position, e);
-			}
-		} else {
-			inner.add(position, e);
-		}
-	}
-
-	public E last() {
-		return inner.get(inner.size() - 1);
-	}
-
-	public boolean isFull() {
-		return inner.size() == capacity;
-	}
-
-	public List<E> asList() {
-		return inner;
-	}
-}

commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java

@@ -2,32 +2,28 @@ package com.commafeed.backend.dao;
 
 import java.time.Instant;
 import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
 
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.builder.CompareToBuilder;
 import org.hibernate.SessionFactory;
 
 import com.commafeed.CommaFeedConfiguration;
-import com.commafeed.backend.FixedSizeSortedList;
 import com.commafeed.backend.feed.FeedEntryKeyword;
 import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
 import com.commafeed.backend.model.FeedEntry;
-import com.commafeed.backend.model.FeedEntryContent;
 import com.commafeed.backend.model.FeedEntryStatus;
 import com.commafeed.backend.model.FeedEntryTag;
 import com.commafeed.backend.model.FeedSubscription;
-import com.commafeed.backend.model.Models;
+import com.commafeed.backend.model.QFeed;
 import com.commafeed.backend.model.QFeedEntry;
 import com.commafeed.backend.model.QFeedEntryContent;
 import com.commafeed.backend.model.QFeedEntryStatus;
 import com.commafeed.backend.model.QFeedEntryTag;
+import com.commafeed.backend.model.QFeedSubscription;
 import com.commafeed.backend.model.User;
 import com.commafeed.backend.model.UserSettings.ReadingOrder;
 import com.commafeed.frontend.model.UnreadCount;
 import com.google.common.collect.Iterables;
-import com.google.common.collect.Ordering;
 import com.querydsl.core.BooleanBuilder;
 import com.querydsl.core.Tuple;
 import com.querydsl.jpa.impl.JPAQuery;
@@ -38,29 +34,19 @@ import jakarta.inject.Singleton;
 @Singleton
 public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 
-	private static final Comparator<FeedEntryStatus> STATUS_COMPARATOR_DESC = (o1, o2) -> {
-		CompareToBuilder builder = new CompareToBuilder();
-		builder.append(o2.getEntryUpdated(), o1.getEntryUpdated());
-		builder.append(o2.getId(), o1.getId());
-		return builder.toComparison();
-	};
-
-	private static final Comparator<FeedEntryStatus> STATUS_COMPARATOR_ASC = Ordering.from(STATUS_COMPARATOR_DESC).reverse();
-
-	private final FeedEntryDAO feedEntryDAO;
 	private final FeedEntryTagDAO feedEntryTagDAO;
 	private final CommaFeedConfiguration config;
 
 	private final QFeedEntryStatus status = QFeedEntryStatus.feedEntryStatus;
 	private final QFeedEntry entry = QFeedEntry.feedEntry;
+	private final QFeed feed = QFeed.feed;
+	private final QFeedSubscription subscription = QFeedSubscription.feedSubscription;
 	private final QFeedEntryContent content = QFeedEntryContent.feedEntryContent;
 	private final QFeedEntryTag entryTag = QFeedEntryTag.feedEntryTag;
 
 	@Inject
-	public FeedEntryStatusDAO(SessionFactory sessionFactory, FeedEntryDAO feedEntryDAO, FeedEntryTagDAO feedEntryTagDAO,
+	public FeedEntryStatusDAO(SessionFactory sessionFactory, FeedEntryTagDAO feedEntryTagDAO, CommaFeedConfiguration config) {
-			CommaFeedConfiguration config) {
 		super(sessionFactory);
-		this.feedEntryDAO = feedEntryDAO;
 		this.feedEntryTagDAO = feedEntryTagDAO;
 		this.config = config;
 	}
@@ -71,6 +57,9 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 		return handleStatus(user, status, sub, entry);
 	}
 
+	/**
+	 * creates an artificial "unread" status if status is null
+	 */
 	private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
 		if (status == null) {
 			Instant unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
@@ -93,6 +82,10 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 	public List<FeedEntryStatus> findStarred(User user, Instant newerThan, int offset, int limit, ReadingOrder order,
 			boolean includeContent) {
 		JPAQuery<FeedEntryStatus> query = query().selectFrom(status).where(status.user.eq(user), status.starred.isTrue());
+		if (includeContent) {
+			query.join(status.entry.content).fetchJoin();
+		}
+
 		if (newerThan != null) {
 			query.where(status.entryInserted.gt(newerThan));
 		}
@@ -115,21 +108,30 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 
 		List<FeedEntryStatus> statuses = query.fetch();
 		for (FeedEntryStatus status : statuses) {
-			status = handleStatus(user, status, status.getSubscription(), status.getEntry());
+			status.setMarkable(true);
-			fetchTags(user, status);
+
+			if (includeContent) {
+				fetchTags(user, status);
+			}
 		}
-		return lazyLoadContent(includeContent, statuses);
+
+		return statuses;
 	}
 
-	private JPAQuery<FeedEntry> buildQuery(User user, FeedSubscription sub, boolean unreadOnly, List<FeedEntryKeyword> keywords,
+	public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
-			Instant newerThan, int offset, int limit, ReadingOrder order, FeedEntryStatus last, String tag, Long minEntryId,
+			List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
-			Long maxEntryId) {
+			boolean onlyIds, String tag, Long minEntryId, Long maxEntryId) {
 
-		JPAQuery<FeedEntry> query = query().selectFrom(entry).where(entry.feed.eq(sub.getFeed()));
+		JPAQuery<Tuple> query = query().select(entry, subscription, status).from(entry);
+		query.join(entry.feed, feed);
+		query.join(subscription).on(subscription.feed.eq(feed).and(subscription.user.eq(user)));
+		query.leftJoin(status).on(status.entry.eq(entry).and(status.subscription.eq(subscription)));
+		query.where(subscription.in(subs));
 
+		if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
+			query.join(entry.content, content).fetchJoin();
+		}
 		if (CollectionUtils.isNotEmpty(keywords)) {
-			query.join(entry.content, content);
-
 			for (FeedEntryKeyword keyword : keywords) {
 				BooleanBuilder or = new BooleanBuilder();
 				or.or(content.content.containsIgnoreCase(keyword.getKeyword()));
@@ -140,18 +142,9 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 				query.where(or);
 			}
 		}
-		query.leftJoin(entry.statuses, status).on(status.subscription.id.eq(sub.getId()));
 
 		if (unreadOnly && tag == null) {
-			BooleanBuilder or = new BooleanBuilder();
+			query.where(buildUnreadPredicate());
-			or.or(status.read.isNull());
-			or.or(status.read.isFalse());
-			query.where(or);
-
-			Instant unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
-			if (unreadThreshold != null) {
-				query.where(entry.updated.goe(unreadThreshold));
-			}
 		}
 
 		if (tag != null) {
@@ -173,14 +166,6 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 			query.where(entry.id.lt(maxEntryId));
 		}
 
-		if (last != null) {
-			if (order == ReadingOrder.desc) {
-				query.where(entry.updated.gt(last.getEntryUpdated()));
-			} else {
-				query.where(entry.updated.lt(last.getEntryUpdated()));
-			}
-		}
-
 		if (order != null) {
 			if (order == ReadingOrder.asc) {
 				query.orderBy(entry.updated.asc(), entry.id.asc());
@@ -198,91 +183,53 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
 		}
 
 		setTimeout(query, config.getApplicationSettings().getQueryTimeout());
-		return query;
-	}
 
-	public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
+		List<FeedEntryStatus> statuses = new ArrayList<>();
-			List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
+		List<Tuple> tuples = query.fetch();
-			boolean onlyIds, String tag, Long minEntryId, Long maxEntryId) {
+		for (Tuple tuple : tuples) {
-		int capacity = offset + limit;
+			FeedEntry e = tuple.get(entry);
+			FeedSubscription sub = tuple.get(subscription);
+			FeedEntryStatus s = handleStatus(user, tuple.get(status), sub, e);
 
-		Comparator<FeedEntryStatus> comparator = order == ReadingOrder.desc ? STATUS_COMPARATOR_DESC : STATUS_COMPARATOR_ASC;
+			if (includeContent) {
-
+				fetchTags(user, s);
-		FixedSizeSortedList<FeedEntryStatus> fssl = new FixedSizeSortedList<>(capacity, comparator);
-		for (FeedSubscription sub : subs) {
-			FeedEntryStatus last = (order != null && fssl.isFull()) ? fssl.last() : null;
-			JPAQuery<FeedEntry> query = buildQuery(user, sub, unreadOnly, keywords, newerThan, -1, capacity, order, last, tag, minEntryId,
-					maxEntryId);
-			List<Tuple> tuples = query.select(entry.id, entry.updated, status.id, entry.content.title).fetch();
-
-			for (Tuple tuple : tuples) {
-				Long id = tuple.get(entry.id);
-				Instant updated = tuple.get(entry.updated);
-				Long statusId = tuple.get(status.id);
-
-				FeedEntryContent content = new FeedEntryContent();
-				content.setTitle(tuple.get(entry.content.title));
-
-				FeedEntry entry = new FeedEntry();
-				entry.setId(id);
-				entry.setUpdated(updated);
-				entry.setContent(content);
-
-				FeedEntryStatus status = new FeedEntryStatus();
-				status.setId(statusId);
-				status.setEntryUpdated(updated);
-				status.setEntry(entry);
-				status.setSubscription(sub);
-
-				fssl.add(status);
 			}
+
+			statuses.add(s);
 		}
 
-		List<FeedEntryStatus> placeholders = fssl.asList();
-		int size = placeholders.size();
-		if (size < offset) {
-			return new ArrayList<>();
-		}
-		placeholders = placeholders.subList(Math.max(offset, 0), size);
-
-		List<FeedEntryStatus> statuses;
-		if (onlyIds) {
-			statuses = placeholders;
-		} else {
-			statuses = new ArrayList<>();
-			for (FeedEntryStatus placeholder : placeholders) {
-				Long statusId = placeholder.getId();
-				FeedEntry entry = feedEntryDAO.findById(placeholder.getEntry().getId());
-				FeedEntryStatus status = handleStatus(user, statusId == null ? null : findById(statusId), placeholder.getSubscription(),
-						entry);
-				status = fetchTags(user, status);
-				statuses.add(status);
-			}
-			statuses = lazyLoadContent(includeContent, statuses);
-		}
 		return statuses;
 	}
 
-	public UnreadCount getUnreadCount(User user, FeedSubscription subscription) {
+	public UnreadCount getUnreadCount(User user, FeedSubscription sub) {
-		UnreadCount uc = null;
+		JPAQuery<Tuple> query = query().select(entry.count(), entry.updated.max())
-		JPAQuery<FeedEntry> query = buildQuery(user, subscription, true, null, null, -1, -1, null, null, null, null, null);
+				.from(entry)
-		List<Tuple> tuples = query.select(entry.count(), entry.updated.max()).fetch();
+				.join(entry.feed, feed)
-		for (Tuple tuple : tuples) {
+				.join(subscription)
-			Long count = tuple.get(entry.count());
+				.on(subscription.feed.eq(feed).and(subscription.user.eq(user)))
-			Instant updated = tuple.get(entry.updated.max());
+				.leftJoin(status)
-			uc = new UnreadCount(subscription.getId(), count == null ? 0 : count, updated);
+				.on(status.entry.eq(entry).and(status.subscription.eq(subscription)))
-		}
+				.where(subscription.eq(sub));
-		return uc;
+
+		query.where(buildUnreadPredicate());
+
+		Tuple tuple = query.fetchOne();
+		Long count = tuple.get(entry.count());
+		Instant updated = tuple.get(entry.updated.max());
+		return new UnreadCount(sub.getId(), count == null ? 0 : count, updated);
 	}
 
-	private List<FeedEntryStatus> lazyLoadContent(boolean includeContent, List<FeedEntryStatus> results) {
+	private BooleanBuilder buildUnreadPredicate() {
-		if (includeContent) {
+		BooleanBuilder or = new BooleanBuilder();
-			for (FeedEntryStatus status : results) {
+		or.or(status.read.isNull());
-				Models.initialize(status.getSubscription().getFeed());
+		or.or(status.read.isFalse());
-				Models.initialize(status.getEntry().getContent());
+
-			}
+		Instant unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
+		if (unreadThreshold != null) {
+			return or.and(entry.updated.goe(unreadThreshold));
+		} else {
+			return or;
 		}
-		return results;
 	}
 
 	public long deleteOldStatuses(Instant olderThan, int limit) {

commafeed-server/src/test/java/com/commafeed/backend/FixedSizeSortedListTest.java

@@ -1,90 +0,0 @@
-package com.commafeed.backend;
-
-import java.util.Comparator;
-
-import org.apache.commons.lang3.ObjectUtils;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-class FixedSizeSortedListTest {
-
-	private static final Comparator<String> COMP = ObjectUtils::compare;
-
-	private FixedSizeSortedList<String> list;
-
-	@BeforeEach
-	public void init() {
-		list = new FixedSizeSortedList<>(3, COMP);
-	}
-
-	@Test
-	void testSimpleAdd() {
-		list.add("0");
-		list.add("1");
-		list.add("2");
-
-		Assertions.assertEquals("0", list.asList().get(0));
-		Assertions.assertEquals("1", list.asList().get(1));
-		Assertions.assertEquals("2", list.asList().get(2));
-	}
-
-	@Test
-	void testIsFull() {
-		list.add("0");
-		list.add("1");
-
-		Assertions.assertFalse(list.isFull());
-		list.add("2");
-		Assertions.assertTrue(list.isFull());
-	}
-
-	@Test
-	void testOrder() {
-		list.add("2");
-		list.add("1");
-		list.add("0");
-
-		Assertions.assertEquals("0", list.asList().get(0));
-		Assertions.assertEquals("1", list.asList().get(1));
-		Assertions.assertEquals("2", list.asList().get(2));
-	}
-
-	@Test
-	void testEviction() {
-		list.add("7");
-		list.add("8");
-		list.add("9");
-
-		list.add("0");
-		list.add("1");
-		list.add("2");
-
-		Assertions.assertEquals("0", list.asList().get(0));
-		Assertions.assertEquals("1", list.asList().get(1));
-		Assertions.assertEquals("2", list.asList().get(2));
-	}
-
-	@Test
-	void testCapacity() {
-		list.add("0");
-		list.add("1");
-		list.add("2");
-		list.add("3");
-
-		Assertions.assertEquals(3, list.asList().size());
-	}
-
-	@Test
-	void testLast() {
-		list.add("0");
-		list.add("1");
-		list.add("2");
-
-		Assertions.assertEquals("2", list.last());
-
-		list.add("3");
-
-		Assertions.assertEquals("2", list.last());
-	}
-}