garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

configurable feed refresh interval

Browse Source
This commit is contained in:
Athou
2013-07-29 10:33:18 +02:00
parent ef79cf1748
commit 5fe5b97130
7 changed files with 50 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/java/com/commafeed/backend/dao/FeedDAO.java
View File

@@ -38,30 +38,30 @@ public class FeedDAO extends GenericDAO<Feed> {
public List<Feed> feeds;
}
private Predicate getUpdatablePredicate(Root<Feed> root, Date threshold) {
private Predicate getUpdatablePredicate(Root<Feed> root) {
Predicate isNull = builder.isNull(root.get(Feed_.disabledUntil));
Predicate lessThan = builder.lessThan(root.get(Feed_.disabledUntil), threshold);
Predicate lessThan = builder.lessThan(root.get(Feed_.disabledUntil), new Date());
return builder.or(isNull, lessThan);
}
public Long getUpdatableCount(Date threshold) {
public Long getUpdatableCount() {
CriteriaQuery<Long> query = builder.createQuery(Long.class);
Root<Feed> root = query.from(getType());
query.select(builder.count(root));
query.where(getUpdatablePredicate(root, threshold));
query.where(getUpdatablePredicate(root));
TypedQuery<Long> q = em.createQuery(query);
return q.getSingleResult();
}
public List<Feed> findNextUpdatable(int count, Date threshold) {
public List<Feed> findNextUpdatable(int count) {
CriteriaQuery<Feed> query = builder.createQuery(getType());
Root<Feed> root = query.from(getType());
query.where(getUpdatablePredicate(root, threshold));
query.where(getUpdatablePredicate(root));
query.orderBy(builder.asc(root.get(Feed_.disabledUntil)));
TypedQuery<Feed> q = em.createQuery(query);

14
src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java
View File

@@ -122,21 +122,15 @@ public class FeedRefreshTaskGiver {
}
public Long getUpdatableCount() {
return feedDAO.getUpdatableCount(getThreshold());
}
private Date getThreshold() {
boolean heavyLoad = applicationSettingsService.get().isHeavyLoad();
Date threshold = DateUtils.addMinutes(new Date(), heavyLoad ? -15 : -5);
return threshold;
return feedDAO.getUpdatableCount();
}
/**
* add a feed to the refresh queue
*/
public void add(Feed feed) {
Date threshold = getThreshold();
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().before(threshold)) {
int refreshInterval = applicationSettingsService.get().getRefreshIntervalMinutes();
if (feed.getLastUpdated() == null || feed.getLastUpdated().before(DateUtils.addMinutes(new Date(), -1 * refreshInterval))) {
addQueue.add(feed);
}
}
@@ -152,7 +146,7 @@ public class FeedRefreshTaskGiver {
if (applicationSettingsService.get().isCrawlingPaused()) {
feeds = Lists.newArrayList();
} else {
feeds = feedDAO.findNextUpdatable(count, getThreshold());
feeds = feedDAO.findNextUpdatable(count);
}
// then, add to those the feeds we got from the add() method. We add them at the beginning of the list as they probably have a

16
src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java
View File

@@ -9,6 +9,7 @@ import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -20,7 +21,6 @@ import com.commafeed.backend.model.ApplicationSettings;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.services.ApplicationSettingsService;
import com.sun.syndication.io.FeedException;
/**
* Calls {@link FeedFetcher} and handles its outcome
@@ -95,6 +95,8 @@ public class FeedRefreshWorker {
}
private void update(Feed feed) {
int refreshInterval = applicationSettingsService.get().getRefreshIntervalMinutes();
Date disabledUntil = DateUtils.addMinutes(new Date(), refreshInterval);
try {
FetchedFeed fetchedFeed = fetcher.fetch(feed.getUrl(), false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
@@ -102,10 +104,9 @@ public class FeedRefreshWorker {
// thrown
List<FeedEntry> entries = fetchedFeed.getEntries();
Date disabledUntil = new Date();
if (applicationSettingsService.get().isHeavyLoad()) {
disabledUntil = FeedUtils.buildDisabledUntil(fetchedFeed.getFeed().getLastEntryDate(), fetchedFeed.getFeed()
.getAverageEntryInterval());
.getAverageEntryInterval(), disabledUntil);
}
feed.setLink(fetchedFeed.getFeed().getLink());
@@ -126,9 +127,8 @@ public class FeedRefreshWorker {
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
Date disabledUntil = null;
if (applicationSettingsService.get().isHeavyLoad()) {
disabledUntil = FeedUtils.buildDisabledUntil(feed.getLastEntryDate(), feed.getAverageEntryInterval());
disabledUntil = FeedUtils.buildDisabledUntil(feed.getLastEntryDate(), feed.getAverageEntryInterval(), disabledUntil);
}
feed.setErrorCount(0);
feed.setMessage(null);
@@ -137,11 +137,7 @@ public class FeedRefreshWorker {
taskGiver.giveBack(feed);
} catch (Exception e) {
String message = "Unable to refresh feed " + feed.getUrl() + " : " + e.getMessage();
if (e instanceof FeedException) {
log.debug(e.getClass().getName() + " " + message, e);
} else {
log.debug(e.getClass().getName() + " " + message, e);
}
log.debug(e.getClass().getName() + " " + message, e);
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage(message);

13
src/main/java/com/commafeed/backend/feeds/FeedUtils.java
View File

@@ -288,7 +288,7 @@ public class FeedUtils {
/**
* When the feed was refreshed successfully
*/
public static Date buildDisabledUntil(Date publishedDate, Long averageEntryInterval) {
public static Date buildDisabledUntil(Date publishedDate, Long averageEntryInterval, Date defaultRefreshInterval) {
Date now = new Date();
if (publishedDate == null) {
@@ -304,9 +304,16 @@ public class FeedUtils {
// older than a week, recheck in 6 hours
return DateUtils.addHours(now, 6);
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
return new Date(Math.min(DateUtils.addHours(now, 6).getTime(), now.getTime() + averageEntryInterval / factor));
// not more than 6 hours
long date = Math.min(DateUtils.addHours(now, 6).getTime(), now.getTime() + averageEntryInterval / factor);
// not less than default refresh interval
date = Math.max(defaultRefreshInterval.getTime(), date);
return new Date(date);
} else {
// unknown case, recheck in 24 hours
return DateUtils.addHours(now, 24);

9
src/main/java/com/commafeed/backend/model/ApplicationSettings.java
View File

@@ -36,6 +36,7 @@ public class ApplicationSettings extends AbstractModel {
private int queryTimeout;
private boolean crawlingPaused;
private int keepStatusDays = 0;
private int refreshIntervalMinutes;
@Column(length = 255)
private String announcement;
@@ -210,4 +211,12 @@ public class ApplicationSettings extends AbstractModel {
this.keepStatusDays = keepStatusDays;
}
public int getRefreshIntervalMinutes() {
return refreshIntervalMinutes;
}
public void setRefreshIntervalMinutes(int refreshIntervalMinutes) {
this.refreshIntervalMinutes = refreshIntervalMinutes;
}
}

8
src/main/resources/changelogs/db.changelog-1.2.xml
View File

@@ -120,5 +120,13 @@
<dropIndex tableName="FEEDENTRIES" indexName="guidHash_index" />
</changeSet>
<changeSet author="athou" id="refresh-interval-setting">
<addColumn tableName="APPLICATIONSETTINGS">
<column name="refreshIntervalMinutes" type="INT"></column>
</addColumn>
<update tableName="APPLICATIONSETTINGS">
<column name="refreshIntervalMinutes" valueNumeric="5"></column>
</update>
</changeSet>
</databaseChangeLog>

7
src/main/webapp/templates/admin.settings.html
View File

@@ -171,6 +171,13 @@
<span class="help-inline">0 = keep forever</span>
</div>
</div>
<div class="control-group">
<label class="control-label" for="refreshIntervalMinutes">Refresh feeds every (minutes)</label>
<div class="controls">
<input type="number" name="refreshIntervalMinutes" class="input-block-level" min="1"
ng-model="settings.refreshIntervalMinutes" />
</div>
</div>
<div class="control-group">
<label class="control-label" for="crawlingPaused">Pause crawling</label>
<div class="controls">