diff --git a/src/main/java/com/commafeed/backend/dao/FeedDAO.java b/src/main/java/com/commafeed/backend/dao/FeedDAO.java index 4b08b7f2..b72f6776 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedDAO.java @@ -38,30 +38,30 @@ public class FeedDAO extends GenericDAO { public List feeds; } - private Predicate getUpdatablePredicate(Root root, Date threshold) { + private Predicate getUpdatablePredicate(Root root) { Predicate isNull = builder.isNull(root.get(Feed_.disabledUntil)); - Predicate lessThan = builder.lessThan(root.get(Feed_.disabledUntil), threshold); + Predicate lessThan = builder.lessThan(root.get(Feed_.disabledUntil), new Date()); return builder.or(isNull, lessThan); } - public Long getUpdatableCount(Date threshold) { + public Long getUpdatableCount() { CriteriaQuery query = builder.createQuery(Long.class); Root root = query.from(getType()); query.select(builder.count(root)); - query.where(getUpdatablePredicate(root, threshold)); + query.where(getUpdatablePredicate(root)); TypedQuery q = em.createQuery(query); return q.getSingleResult(); } - public List findNextUpdatable(int count, Date threshold) { + public List findNextUpdatable(int count) { CriteriaQuery query = builder.createQuery(getType()); Root root = query.from(getType()); - query.where(getUpdatablePredicate(root, threshold)); + query.where(getUpdatablePredicate(root)); query.orderBy(builder.asc(root.get(Feed_.disabledUntil))); TypedQuery q = em.createQuery(query); diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java index 870c0086..41152084 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java @@ -122,21 +122,15 @@ public class FeedRefreshTaskGiver { } public Long getUpdatableCount() { - return feedDAO.getUpdatableCount(getThreshold()); - } - - private Date getThreshold() { - boolean heavyLoad = applicationSettingsService.get().isHeavyLoad(); - Date threshold = DateUtils.addMinutes(new Date(), heavyLoad ? -15 : -5); - return threshold; + return feedDAO.getUpdatableCount(); } /** * add a feed to the refresh queue */ public void add(Feed feed) { - Date threshold = getThreshold(); - if (feed.getDisabledUntil() == null || feed.getDisabledUntil().before(threshold)) { + int refreshInterval = applicationSettingsService.get().getRefreshIntervalMinutes(); + if (feed.getLastUpdated() == null || feed.getLastUpdated().before(DateUtils.addMinutes(new Date(), -1 * refreshInterval))) { addQueue.add(feed); } } @@ -152,7 +146,7 @@ public class FeedRefreshTaskGiver { if (applicationSettingsService.get().isCrawlingPaused()) { feeds = Lists.newArrayList(); } else { - feeds = feedDAO.findNextUpdatable(count, getThreshold()); + feeds = feedDAO.findNextUpdatable(count); } // then, add to those the feeds we got from the add() method. We add them at the beginning of the list as they probably have a diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java index 24a00fac..41a24fb8 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java @@ -9,6 +9,7 @@ import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang.time.DateUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +21,6 @@ import com.commafeed.backend.model.ApplicationSettings; import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.services.ApplicationSettingsService; -import com.sun.syndication.io.FeedException; /** * Calls {@link FeedFetcher} and handles its outcome @@ -95,6 +95,8 @@ public class FeedRefreshWorker { } private void update(Feed feed) { + int refreshInterval = applicationSettingsService.get().getRefreshIntervalMinutes(); + Date disabledUntil = DateUtils.addMinutes(new Date(), refreshInterval); try { FetchedFeed fetchedFeed = fetcher.fetch(feed.getUrl(), false, feed.getLastModifiedHeader(), feed.getEtagHeader(), feed.getLastPublishedDate(), feed.getLastContentHash()); @@ -102,10 +104,9 @@ public class FeedRefreshWorker { // thrown List entries = fetchedFeed.getEntries(); - Date disabledUntil = new Date(); if (applicationSettingsService.get().isHeavyLoad()) { disabledUntil = FeedUtils.buildDisabledUntil(fetchedFeed.getFeed().getLastEntryDate(), fetchedFeed.getFeed() - .getAverageEntryInterval()); + .getAverageEntryInterval(), disabledUntil); } feed.setLink(fetchedFeed.getFeed().getLink()); @@ -126,9 +127,8 @@ public class FeedRefreshWorker { } catch (NotModifiedException e) { log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage()); - Date disabledUntil = null; if (applicationSettingsService.get().isHeavyLoad()) { - disabledUntil = FeedUtils.buildDisabledUntil(feed.getLastEntryDate(), feed.getAverageEntryInterval()); + disabledUntil = FeedUtils.buildDisabledUntil(feed.getLastEntryDate(), feed.getAverageEntryInterval(), disabledUntil); } feed.setErrorCount(0); feed.setMessage(null); @@ -137,11 +137,7 @@ public class FeedRefreshWorker { taskGiver.giveBack(feed); } catch (Exception e) { String message = "Unable to refresh feed " + feed.getUrl() + " : " + e.getMessage(); - if (e instanceof FeedException) { - log.debug(e.getClass().getName() + " " + message, e); - } else { - log.debug(e.getClass().getName() + " " + message, e); - } + log.debug(e.getClass().getName() + " " + message, e); feed.setErrorCount(feed.getErrorCount() + 1); feed.setMessage(message); diff --git a/src/main/java/com/commafeed/backend/feeds/FeedUtils.java b/src/main/java/com/commafeed/backend/feeds/FeedUtils.java index ef716429..b2bece61 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedUtils.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedUtils.java @@ -288,7 +288,7 @@ public class FeedUtils { /** * When the feed was refreshed successfully */ - public static Date buildDisabledUntil(Date publishedDate, Long averageEntryInterval) { + public static Date buildDisabledUntil(Date publishedDate, Long averageEntryInterval, Date defaultRefreshInterval) { Date now = new Date(); if (publishedDate == null) { @@ -304,9 +304,16 @@ public class FeedUtils { // older than a week, recheck in 6 hours return DateUtils.addHours(now, 6); } else if (averageEntryInterval != null) { - // use average time between entries to decide when to refresh next + // use average time between entries to decide when to refresh next, divided by factor int factor = 2; - return new Date(Math.min(DateUtils.addHours(now, 6).getTime(), now.getTime() + averageEntryInterval / factor)); + + // not more than 6 hours + long date = Math.min(DateUtils.addHours(now, 6).getTime(), now.getTime() + averageEntryInterval / factor); + + // not less than default refresh interval + date = Math.max(defaultRefreshInterval.getTime(), date); + + return new Date(date); } else { // unknown case, recheck in 24 hours return DateUtils.addHours(now, 24); diff --git a/src/main/java/com/commafeed/backend/model/ApplicationSettings.java b/src/main/java/com/commafeed/backend/model/ApplicationSettings.java index 618a6f32..e9964bde 100644 --- a/src/main/java/com/commafeed/backend/model/ApplicationSettings.java +++ b/src/main/java/com/commafeed/backend/model/ApplicationSettings.java @@ -36,6 +36,7 @@ public class ApplicationSettings extends AbstractModel { private int queryTimeout; private boolean crawlingPaused; private int keepStatusDays = 0; + private int refreshIntervalMinutes; @Column(length = 255) private String announcement; @@ -210,4 +211,12 @@ public class ApplicationSettings extends AbstractModel { this.keepStatusDays = keepStatusDays; } + public int getRefreshIntervalMinutes() { + return refreshIntervalMinutes; + } + + public void setRefreshIntervalMinutes(int refreshIntervalMinutes) { + this.refreshIntervalMinutes = refreshIntervalMinutes; + } + } diff --git a/src/main/resources/changelogs/db.changelog-1.2.xml b/src/main/resources/changelogs/db.changelog-1.2.xml index 9da0843b..477112e7 100644 --- a/src/main/resources/changelogs/db.changelog-1.2.xml +++ b/src/main/resources/changelogs/db.changelog-1.2.xml @@ -120,5 +120,13 @@ + + + + + + + + diff --git a/src/main/webapp/templates/admin.settings.html b/src/main/webapp/templates/admin.settings.html index 5ae9b83b..7c8481bd 100644 --- a/src/main/webapp/templates/admin.settings.html +++ b/src/main/webapp/templates/admin.settings.html @@ -171,6 +171,13 @@ 0 = keep forever +
+ +
+ +
+