Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-03-21 21:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactored the way we handle feed refresh queue

Browse Source
This commit is contained in:
Athou
2013-07-27 15:45:03 +02:00
parent 1131d70645
commit 3dd4f140e2
8 changed files with 29 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/com/commafeed/backend/feeds/FeedParser.java
View File

@@ -49,7 +49,6 @@ public class FeedParser {
FetchedFeed fetchedFeed = new FetchedFeed();
Feed feed = fetchedFeed.getFeed();
List<FeedEntry> entries = fetchedFeed.getEntries();
feed.setLastUpdated(new Date());
try {
String encoding = FeedUtils.guessEncoding(xml);

8
src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java
View File

@@ -129,14 +129,12 @@ public class FeedRefreshTaskGiver {
public void add(Feed feed) {
Date threshold = getThreshold();
if (feed.getLastUpdated() == null || feed.getLastUpdated().before(threshold)) {
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().before(threshold)) {
addQueue.add(feed);
}
}
private void refill() {
Date now = new Date();
int count = Math.min(100, 3 * backgroundThreads);
List<Feed> feeds = null;
if (applicationSettingsService.get().isCrawlingPaused()) {
@@ -152,7 +150,7 @@ public class FeedRefreshTaskGiver {
Map<Long, Feed> map = Maps.newLinkedHashMap();
for (Feed f : feeds) {
f.setLastUpdated(now);
f.setDisabledUntil(new Date());
map.put(f.getId(), f);
}
takeQueue.addAll(map.values());
@@ -160,7 +158,6 @@ public class FeedRefreshTaskGiver {
size = giveBackQueue.size();
for (int i = 0; i < size; i++) {
Feed f = giveBackQueue.poll();
f.setLastUpdated(now);
map.put(f.getId(), f);
}
@@ -171,6 +168,7 @@ public class FeedRefreshTaskGiver {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setLastUpdated(new Date());
giveBackQueue.add(feed);
}

3
src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java
View File

@@ -129,7 +129,8 @@ public class FeedRefreshUpdater {
handlePubSub(feed);
}
if (!ok) {
feed.setDisabledUntil(null);
// requeue asap
feed.setDisabledUntil(new Date(0));
}
metricsBean.feedUpdated();
taskGiver.giveBack(feed);

4
src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java
View File

@@ -95,7 +95,6 @@ public class FeedRefreshWorker {
}
private void update(Feed feed) {
Date now = new Date();
try {
FetchedFeed fetchedFeed = fetcher.fetch(feed.getUrl(), false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
@@ -103,13 +102,12 @@ public class FeedRefreshWorker {
// thrown
List<FeedEntry> entries = fetchedFeed.getEntries();
Date disabledUntil = null;
Date disabledUntil = new Date();
if (applicationSettingsService.get().isHeavyLoad()) {
disabledUntil = FeedUtils.buildDisabledUntil(fetchedFeed.getFeed().getLastEntryDate(), fetchedFeed.getFeed()
.getAverageEntryInterval());
}
feed.setLastUpdateSuccess(now);
feed.setLink(fetchedFeed.getFeed().getLink());
feed.setLastModifiedHeader(fetchedFeed.getFeed().getLastModifiedHeader());
feed.setEtagHeader(fetchedFeed.getFeed().getEtagHeader());