diff --git a/src/main/java/com/commafeed/backend/dao/FeedDAO.java b/src/main/java/com/commafeed/backend/dao/FeedDAO.java index 3af5f1e1..949bd39b 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedDAO.java @@ -39,18 +39,12 @@ public class FeedDAO extends GenericDAO { public List feeds; } - private List getUpdatablePredicates(Root root, Date threshold) { + private Predicate getUpdatablePredicate(Root root, Date threshold) { - Predicate hasSubscriptions = builder.isNotEmpty(root.get(Feed_.subscriptions)); + Predicate isNull = builder.isNull(root.get(Feed_.disabledUntil)); + Predicate lessThan = builder.lessThan(root.get(Feed_.disabledUntil), threshold); - Predicate neverUpdated = builder.isNull(root.get(Feed_.lastUpdated)); - Predicate updatedBeforeThreshold = builder.lessThan(root.get(Feed_.lastUpdated), threshold); - - Predicate disabledDateIsNull = builder.isNull(root.get(Feed_.disabledUntil)); - Predicate disabledDateIsInPast = builder.lessThan(root.get(Feed_.disabledUntil), new Date()); - - return Lists.newArrayList(hasSubscriptions, builder.or(neverUpdated, updatedBeforeThreshold), - builder.or(disabledDateIsNull, disabledDateIsInPast)); + return builder.or(isNull, lessThan); } public Long getUpdatableCount(Date threshold) { @@ -58,7 +52,7 @@ public class FeedDAO extends GenericDAO { Root root = query.from(getType()); query.select(builder.count(root)); - query.where(getUpdatablePredicates(root, threshold).toArray(new Predicate[0])); + query.where(getUpdatablePredicate(root, threshold)); TypedQuery q = em.createQuery(query); return q.getSingleResult(); @@ -68,9 +62,8 @@ public class FeedDAO extends GenericDAO { CriteriaQuery query = builder.createQuery(getType()); Root root = query.from(getType()); - query.where(getUpdatablePredicates(root, threshold).toArray(new Predicate[0])); - - query.orderBy(builder.asc(root.get(Feed_.lastUpdated))); + query.where(getUpdatablePredicate(root, threshold)); + query.orderBy(builder.asc(root.get(Feed_.disabledUntil))); TypedQuery q = em.createQuery(query); q.setMaxResults(count); diff --git a/src/main/java/com/commafeed/backend/feeds/FeedParser.java b/src/main/java/com/commafeed/backend/feeds/FeedParser.java index 81617b71..c6cff004 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedParser.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedParser.java @@ -49,7 +49,6 @@ public class FeedParser { FetchedFeed fetchedFeed = new FetchedFeed(); Feed feed = fetchedFeed.getFeed(); List entries = fetchedFeed.getEntries(); - feed.setLastUpdated(new Date()); try { String encoding = FeedUtils.guessEncoding(xml); diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java index de689834..cd5cb8f9 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java @@ -129,14 +129,12 @@ public class FeedRefreshTaskGiver { public void add(Feed feed) { Date threshold = getThreshold(); - if (feed.getLastUpdated() == null || feed.getLastUpdated().before(threshold)) { + if (feed.getDisabledUntil() == null || feed.getDisabledUntil().before(threshold)) { addQueue.add(feed); } } private void refill() { - Date now = new Date(); - int count = Math.min(100, 3 * backgroundThreads); List feeds = null; if (applicationSettingsService.get().isCrawlingPaused()) { @@ -152,7 +150,7 @@ public class FeedRefreshTaskGiver { Map map = Maps.newLinkedHashMap(); for (Feed f : feeds) { - f.setLastUpdated(now); + f.setDisabledUntil(new Date()); map.put(f.getId(), f); } takeQueue.addAll(map.values()); @@ -160,7 +158,6 @@ public class FeedRefreshTaskGiver { size = giveBackQueue.size(); for (int i = 0; i < size; i++) { Feed f = giveBackQueue.poll(); - f.setLastUpdated(now); map.put(f.getId(), f); } @@ -171,6 +168,7 @@ public class FeedRefreshTaskGiver { String normalized = FeedUtils.normalizeURL(feed.getUrl()); feed.setNormalizedUrl(normalized); feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized)); + feed.setLastUpdated(new Date()); giveBackQueue.add(feed); } diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java index b8c0c632..65a2f535 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java @@ -129,7 +129,8 @@ public class FeedRefreshUpdater { handlePubSub(feed); } if (!ok) { - feed.setDisabledUntil(null); + // requeue asap + feed.setDisabledUntil(new Date(0)); } metricsBean.feedUpdated(); taskGiver.giveBack(feed); diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java index 663597c9..f670928a 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java @@ -95,7 +95,6 @@ public class FeedRefreshWorker { } private void update(Feed feed) { - Date now = new Date(); try { FetchedFeed fetchedFeed = fetcher.fetch(feed.getUrl(), false, feed.getLastModifiedHeader(), feed.getEtagHeader(), feed.getLastPublishedDate(), feed.getLastContentHash()); @@ -103,13 +102,12 @@ public class FeedRefreshWorker { // thrown List entries = fetchedFeed.getEntries(); - Date disabledUntil = null; + Date disabledUntil = new Date(); if (applicationSettingsService.get().isHeavyLoad()) { disabledUntil = FeedUtils.buildDisabledUntil(fetchedFeed.getFeed().getLastEntryDate(), fetchedFeed.getFeed() .getAverageEntryInterval()); } - feed.setLastUpdateSuccess(now); feed.setLink(fetchedFeed.getFeed().getLink()); feed.setLastModifiedHeader(fetchedFeed.getFeed().getLastModifiedHeader()); feed.setEtagHeader(fetchedFeed.getFeed().getEtagHeader()); diff --git a/src/main/java/com/commafeed/backend/model/Feed.java b/src/main/java/com/commafeed/backend/model/Feed.java index 56aa2570..9e6f5af4 100644 --- a/src/main/java/com/commafeed/backend/model/Feed.java +++ b/src/main/java/com/commafeed/backend/model/Feed.java @@ -62,12 +62,6 @@ public class Feed extends AbstractModel { @Temporal(TemporalType.TIMESTAMP) private Date lastEntryDate; - /** - * Last time we successfully refreshed the feed - */ - @Temporal(TemporalType.TIMESTAMP) - private Date lastUpdateSuccess; - /** * error message while retrieving the feed */ @@ -229,14 +223,6 @@ public class Feed extends AbstractModel { this.etagHeader = etagHeader; } - public Date getLastUpdateSuccess() { - return lastUpdateSuccess; - } - - public void setLastUpdateSuccess(Date lastUpdateSuccess) { - this.lastUpdateSuccess = lastUpdateSuccess; - } - public String getPushHub() { return pushHub; } diff --git a/src/main/java/com/commafeed/backend/services/FeedService.java b/src/main/java/com/commafeed/backend/services/FeedService.java index e3cc8a6e..9ae903e4 100644 --- a/src/main/java/com/commafeed/backend/services/FeedService.java +++ b/src/main/java/com/commafeed/backend/services/FeedService.java @@ -1,5 +1,7 @@ package com.commafeed.backend.services; +import java.util.Date; + import javax.ejb.Lock; import javax.ejb.LockType; import javax.ejb.Singleton; @@ -31,6 +33,7 @@ public class FeedService { feed.setUrlHash(DigestUtils.sha1Hex(url)); feed.setNormalizedUrl(normalized); feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized)); + feed.setDisabledUntil(new Date(0)); feedDAO.saveOrUpdate(feed); } return feed; diff --git a/src/main/resources/changelogs/db.changelog-1.2.xml b/src/main/resources/changelogs/db.changelog-1.2.xml index 34297474..def2bbee 100644 --- a/src/main/resources/changelogs/db.changelog-1.2.xml +++ b/src/main/resources/changelogs/db.changelog-1.2.xml @@ -31,7 +31,7 @@ - + @@ -60,14 +60,14 @@ - + - + @@ -77,19 +77,19 @@ - + - - + + - + @@ -100,5 +100,10 @@ - + + + + + +