configurable feed refresh interval

src/main/java/com/commafeed/backend/dao/FeedDAO.java

@@ -38,30 +38,30 @@ public class FeedDAO extends GenericDAO<Feed> {
 		public List<Feed> feeds;
 	}
 
-	private Predicate getUpdatablePredicate(Root<Feed> root, Date threshold) {
+	private Predicate getUpdatablePredicate(Root<Feed> root) {
 
 		Predicate isNull = builder.isNull(root.get(Feed_.disabledUntil));
-		Predicate lessThan = builder.lessThan(root.get(Feed_.disabledUntil), threshold);
+		Predicate lessThan = builder.lessThan(root.get(Feed_.disabledUntil), new Date());
 
 		return builder.or(isNull, lessThan);
 	}
 
-	public Long getUpdatableCount(Date threshold) {
+	public Long getUpdatableCount() {
 		CriteriaQuery<Long> query = builder.createQuery(Long.class);
 		Root<Feed> root = query.from(getType());
 
 		query.select(builder.count(root));
-		query.where(getUpdatablePredicate(root, threshold));
+		query.where(getUpdatablePredicate(root));
 
 		TypedQuery<Long> q = em.createQuery(query);
 		return q.getSingleResult();
 	}
 
-	public List<Feed> findNextUpdatable(int count, Date threshold) {
+	public List<Feed> findNextUpdatable(int count) {
 		CriteriaQuery<Feed> query = builder.createQuery(getType());
 		Root<Feed> root = query.from(getType());
 
-		query.where(getUpdatablePredicate(root, threshold));
+		query.where(getUpdatablePredicate(root));
 		query.orderBy(builder.asc(root.get(Feed_.disabledUntil)));
 
 		TypedQuery<Feed> q = em.createQuery(query);

src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java

@@ -122,21 +122,15 @@ public class FeedRefreshTaskGiver {
 	}
 
 	public Long getUpdatableCount() {
-		return feedDAO.getUpdatableCount(getThreshold());
+		return feedDAO.getUpdatableCount();
-	}
-
-	private Date getThreshold() {
-		boolean heavyLoad = applicationSettingsService.get().isHeavyLoad();
-		Date threshold = DateUtils.addMinutes(new Date(), heavyLoad ? -15 : -5);
-		return threshold;
 	}
 
 	/**
 	 * add a feed to the refresh queue
 	 */
 	public void add(Feed feed) {
-		Date threshold = getThreshold();
+		int refreshInterval = applicationSettingsService.get().getRefreshIntervalMinutes();
-		if (feed.getDisabledUntil() == null || feed.getDisabledUntil().before(threshold)) {
+		if (feed.getLastUpdated() == null || feed.getLastUpdated().before(DateUtils.addMinutes(new Date(), -1 * refreshInterval))) {
 			addQueue.add(feed);
 		}
 	}
@@ -152,7 +146,7 @@ public class FeedRefreshTaskGiver {
 		if (applicationSettingsService.get().isCrawlingPaused()) {
 			feeds = Lists.newArrayList();
 		} else {
-			feeds = feedDAO.findNextUpdatable(count, getThreshold());
+			feeds = feedDAO.findNextUpdatable(count);
 		}
 
 		// then, add to those the feeds we got from the add() method. We add them at the beginning of the list as they probably have a

src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java

@@ -9,6 +9,7 @@ import javax.enterprise.context.ApplicationScoped;
 import javax.inject.Inject;
 
 import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.time.DateUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -20,7 +21,6 @@ import com.commafeed.backend.model.ApplicationSettings;
 import com.commafeed.backend.model.Feed;
 import com.commafeed.backend.model.FeedEntry;
 import com.commafeed.backend.services.ApplicationSettingsService;
-import com.sun.syndication.io.FeedException;
 
 /**
  * Calls {@link FeedFetcher} and handles its outcome
@@ -95,6 +95,8 @@ public class FeedRefreshWorker {
 	}
 
 	private void update(Feed feed) {
+		int refreshInterval = applicationSettingsService.get().getRefreshIntervalMinutes();
+		Date disabledUntil = DateUtils.addMinutes(new Date(), refreshInterval);
 		try {
 			FetchedFeed fetchedFeed = fetcher.fetch(feed.getUrl(), false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
 					feed.getLastPublishedDate(), feed.getLastContentHash());
@@ -102,10 +104,9 @@ public class FeedRefreshWorker {
 			// thrown
 			List<FeedEntry> entries = fetchedFeed.getEntries();
 
-			Date disabledUntil = new Date();
 			if (applicationSettingsService.get().isHeavyLoad()) {
 				disabledUntil = FeedUtils.buildDisabledUntil(fetchedFeed.getFeed().getLastEntryDate(), fetchedFeed.getFeed()
-						.getAverageEntryInterval());
+						.getAverageEntryInterval(), disabledUntil);
 			}
 
 			feed.setLink(fetchedFeed.getFeed().getLink());
@@ -126,9 +127,8 @@ public class FeedRefreshWorker {
 		} catch (NotModifiedException e) {
 			log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
 
-			Date disabledUntil = null;
 			if (applicationSettingsService.get().isHeavyLoad()) {
-				disabledUntil = FeedUtils.buildDisabledUntil(feed.getLastEntryDate(), feed.getAverageEntryInterval());
+				disabledUntil = FeedUtils.buildDisabledUntil(feed.getLastEntryDate(), feed.getAverageEntryInterval(), disabledUntil);
 			}
 			feed.setErrorCount(0);
 			feed.setMessage(null);
@@ -137,11 +137,7 @@ public class FeedRefreshWorker {
 			taskGiver.giveBack(feed);
 		} catch (Exception e) {
 			String message = "Unable to refresh feed " + feed.getUrl() + " : " + e.getMessage();
-			if (e instanceof FeedException) {
 			log.debug(e.getClass().getName() + " " + message, e);
-			} else {
-				log.debug(e.getClass().getName() + " " + message, e);
-			}
 
 			feed.setErrorCount(feed.getErrorCount() + 1);
 			feed.setMessage(message);

src/main/java/com/commafeed/backend/feeds/FeedUtils.java

@@ -288,7 +288,7 @@ public class FeedUtils {
 	/**
 	 * When the feed was refreshed successfully
 	 */
-	public static Date buildDisabledUntil(Date publishedDate, Long averageEntryInterval) {
+	public static Date buildDisabledUntil(Date publishedDate, Long averageEntryInterval, Date defaultRefreshInterval) {
 		Date now = new Date();
 
 		if (publishedDate == null) {
@@ -304,9 +304,16 @@ public class FeedUtils {
 			// older than a week, recheck in 6 hours
 			return DateUtils.addHours(now, 6);
 		} else if (averageEntryInterval != null) {
-			// use average time between entries to decide when to refresh next
+			// use average time between entries to decide when to refresh next, divided by factor
 			int factor = 2;
-			return new Date(Math.min(DateUtils.addHours(now, 6).getTime(), now.getTime() + averageEntryInterval / factor));
+
+			// not more than 6 hours
+			long date = Math.min(DateUtils.addHours(now, 6).getTime(), now.getTime() + averageEntryInterval / factor);
+
+			// not less than default refresh interval
+			date = Math.max(defaultRefreshInterval.getTime(), date);
+
+			return new Date(date);
 		} else {
 			// unknown case, recheck in 24 hours
 			return DateUtils.addHours(now, 24);

src/main/java/com/commafeed/backend/model/ApplicationSettings.java

@@ -36,6 +36,7 @@ public class ApplicationSettings extends AbstractModel {
 	private int queryTimeout;
 	private boolean crawlingPaused;
 	private int keepStatusDays = 0;
+	private int refreshIntervalMinutes;
 
 	@Column(length = 255)
 	private String announcement;
@@ -210,4 +211,12 @@ public class ApplicationSettings extends AbstractModel {
 		this.keepStatusDays = keepStatusDays;
 	}
 
+	public int getRefreshIntervalMinutes() {
+		return refreshIntervalMinutes;
+	}
+
+	public void setRefreshIntervalMinutes(int refreshIntervalMinutes) {
+		this.refreshIntervalMinutes = refreshIntervalMinutes;
+	}
+
 }

src/main/resources/changelogs/db.changelog-1.2.xml

@@ -120,5 +120,13 @@
 		<dropIndex tableName="FEEDENTRIES" indexName="guidHash_index" />
 	</changeSet>
 	
+	<changeSet author="athou" id="refresh-interval-setting">
+		<addColumn tableName="APPLICATIONSETTINGS">
+			<column name="refreshIntervalMinutes" type="INT"></column>
+		</addColumn>
+		<update tableName="APPLICATIONSETTINGS">
+			<column name="refreshIntervalMinutes" valueNumeric="5"></column>
+		</update>
+	</changeSet>
 
 </databaseChangeLog>

src/main/webapp/templates/admin.settings.html

@@ -171,6 +171,13 @@
 							<span class="help-inline">0 = keep forever</span>
 						</div>
 					</div>
+					<div class="control-group">
+						<label class="control-label" for="refreshIntervalMinutes">Refresh feeds every (minutes)</label>
+						<div class="controls">
+							<input type="number" name="refreshIntervalMinutes" class="input-block-level" min="1"
+								ng-model="settings.refreshIntervalMinutes" />
+						</div>
+					</div>
 					<div class="control-group">
 						<label class="control-label" for="crawlingPaused">Pause crawling</label>
 						<div class="controls">