diff --git a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java index 6feef75a..cf7e6b28 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java @@ -15,6 +15,7 @@ import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntry_; import com.commafeed.backend.model.Feed_; +import com.google.common.collect.Lists; @Stateless public class FeedEntryDAO extends GenericDAO { @@ -30,6 +31,20 @@ public class FeedEntryDAO extends GenericDAO { return q.getResultList(); } + public List findByGuids(List guids) { + List hashes = Lists.newArrayList(); + for (String guid : guids) { + hashes.add(DigestUtils.sha1Hex(guid)); + } + + CriteriaQuery query = builder.createQuery(getType()); + Root root = query.from(getType()); + query.where(root.get(FeedEntry_.guidHash).in(hashes)); + root.fetch(FeedEntry_.feeds, JoinType.LEFT); + TypedQuery q = em.createQuery(query); + return q.getResultList(); + } + public List findByFeed(Feed feed, int offset, int limit) { CriteriaQuery query = builder.createQuery(getType()); Root root = query.from(getType()); diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java index c6b88a8a..3c2c17b6 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java @@ -14,12 +14,14 @@ import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.inject.Singleton; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.commafeed.backend.MetricsBean; import com.commafeed.backend.dao.FeedDAO; +import com.commafeed.backend.dao.FeedEntryDAO; import com.commafeed.backend.dao.FeedSubscriptionDAO; import com.commafeed.backend.model.ApplicationSettings; import com.commafeed.backend.model.Feed; @@ -29,6 +31,7 @@ import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.pubsubhubbub.SubscriptionHandler; import com.commafeed.backend.services.ApplicationSettingsService; import com.commafeed.backend.services.FeedUpdateService; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.Striped; @Singleton @@ -58,6 +61,9 @@ public class FeedRefreshUpdater { @Inject FeedSubscriptionDAO feedSubscriptionDAO; + @Inject + FeedEntryDAO feedEntryDAO; + private ThreadPoolExecutor pool; private BlockingQueue queue; private Striped locks; @@ -113,7 +119,7 @@ public class FeedRefreshUpdater { @Override public void run() { boolean ok = true; - if (entries != null) { + if (CollectionUtils.isNotEmpty(entries) && hasWork()) { List subscriptions = feedSubscriptionDAO .findByFeed(feed); for (FeedEntry entry : entries) { @@ -131,6 +137,26 @@ public class FeedRefreshUpdater { taskGiver.giveBack(feed); } + private boolean hasWork() { + boolean hasWork = false; + + List guids = Lists.newArrayList(); + for (FeedEntry entry : entries) { + guids.add(entry.getGuid()); + } + + List existingEntries = feedEntryDAO.findByGuids(guids); + for (FeedEntry entry : entries) { + FeedEntry foundEntry = FeedUtils.findEntry(existingEntries, + entry); + if (foundEntry == null + || FeedUtils.findFeed(foundEntry.getFeeds(), feed) == null) { + hasWork = true; + break; + } + } + return hasWork; + } } private boolean updateEntry(final Feed feed, final FeedEntry entry, diff --git a/src/main/java/com/commafeed/backend/feeds/FeedUtils.java b/src/main/java/com/commafeed/backend/feeds/FeedUtils.java index 50382d8c..86635540 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedUtils.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedUtils.java @@ -3,11 +3,13 @@ package com.commafeed.backend.feeds; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.Calendar; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.time.DateUtils; import org.apache.commons.math.stat.descriptive.SummaryStatistics; @@ -19,6 +21,7 @@ import org.mozilla.universalchardet.UniversalDetector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedEntry; import com.google.api.client.util.Lists; @@ -93,6 +96,30 @@ public class FeedUtils { return content; } + public static FeedEntry findEntry(Collection list, + FeedEntry entry) { + FeedEntry found = null; + for (FeedEntry e : list) { + if (StringUtils.equals(entry.getGuid(), e.getGuid()) + && StringUtils.equals(entry.getUrl(), e.getUrl())) { + found = e; + break; + } + } + return found; + } + + public static Feed findFeed(Collection list, Feed feed) { + Feed found = null; + for (Feed f : list) { + if (ObjectUtils.equals(feed.getId(), f.getId())) { + found = f; + break; + } + } + return found; + } + public static String trimInvalidXmlCharacters(String xml) { if (StringUtils.isBlank(xml)) { return null; diff --git a/src/main/java/com/commafeed/backend/services/FeedUpdateService.java b/src/main/java/com/commafeed/backend/services/FeedUpdateService.java index 3d639c56..e3959fb2 100644 --- a/src/main/java/com/commafeed/backend/services/FeedUpdateService.java +++ b/src/main/java/com/commafeed/backend/services/FeedUpdateService.java @@ -2,14 +2,10 @@ package com.commafeed.backend.services; import java.util.Calendar; import java.util.List; -import java.util.Set; import javax.ejb.Stateless; import javax.inject.Inject; -import org.apache.commons.lang.ObjectUtils; -import org.apache.commons.lang.StringUtils; - import com.commafeed.backend.MetricsBean; import com.commafeed.backend.dao.FeedEntryDAO; import com.commafeed.backend.dao.FeedEntryStatusDAO; @@ -40,7 +36,7 @@ public class FeedUpdateService { public void updateEntry(Feed feed, FeedEntry entry, List subscriptions) { - FeedEntry foundEntry = findEntry( + FeedEntry foundEntry = FeedUtils.findEntry( feedEntryDAO.findByGuid(entry.getGuid()), entry); FeedEntry update = null; @@ -56,11 +52,9 @@ public class FeedUpdateService { entry.getFeeds().add(feed); update = entry; - } else { - if (!findFeed(foundEntry.getFeeds(), feed)) { - foundEntry.getFeeds().add(feed); - update = foundEntry; - } + } else if (FeedUtils.findFeed(foundEntry.getFeeds(), feed) == null) { + foundEntry.getFeeds().add(feed); + update = foundEntry; } if (update != null) { @@ -77,27 +71,4 @@ public class FeedUpdateService { } } - private FeedEntry findEntry(List existingEntries, FeedEntry entry) { - FeedEntry found = null; - for (FeedEntry existing : existingEntries) { - if (StringUtils.equals(entry.getGuid(), existing.getGuid()) - && StringUtils.equals(entry.getUrl(), existing.getUrl())) { - found = existing; - break; - } - } - return found; - } - - private boolean findFeed(Set feeds, Feed feed) { - boolean found = false; - for (Feed existingFeed : feeds) { - if (ObjectUtils.equals(existingFeed.getId(), feed.getId())) { - found = true; - break; - } - } - return found; - } - }