forked from Archives/Athou_commafeed
added comments for refill()
This commit is contained in:
@@ -93,6 +93,7 @@ public class FeedRefreshTaskGiver {
|
|||||||
worker.updateFeed(feed);
|
worker.updateFeed(feed);
|
||||||
} else {
|
} else {
|
||||||
log.debug("nothing to do, sleeping for 15s");
|
log.debug("nothing to do, sleeping for 15s");
|
||||||
|
metricsBean.threadWaited();
|
||||||
try {
|
try {
|
||||||
Thread.sleep(15000);
|
Thread.sleep(15000);
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
@@ -107,6 +108,9 @@ public class FeedRefreshTaskGiver {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* take a feed from the refresh queue
|
||||||
|
*/
|
||||||
private Feed take() {
|
private Feed take() {
|
||||||
Feed feed = takeQueue.poll();
|
Feed feed = takeQueue.poll();
|
||||||
|
|
||||||
@@ -127,6 +131,9 @@ public class FeedRefreshTaskGiver {
|
|||||||
return threshold;
|
return threshold;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* add a feed to the refresh queue
|
||||||
|
*/
|
||||||
public void add(Feed feed) {
|
public void add(Feed feed) {
|
||||||
Date threshold = getThreshold();
|
Date threshold = getThreshold();
|
||||||
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().before(threshold)) {
|
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().before(threshold)) {
|
||||||
@@ -134,8 +141,13 @@ public class FeedRefreshTaskGiver {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* refills the refresh queue and empties the giveBack queue while at it
|
||||||
|
*/
|
||||||
private void refill() {
|
private void refill() {
|
||||||
int count = Math.min(100, 3 * backgroundThreads);
|
int count = Math.min(100, 3 * backgroundThreads);
|
||||||
|
|
||||||
|
// first, get feeds that are up to refresh from the database
|
||||||
List<Feed> feeds = null;
|
List<Feed> feeds = null;
|
||||||
if (applicationSettingsService.get().isCrawlingPaused()) {
|
if (applicationSettingsService.get().isCrawlingPaused()) {
|
||||||
feeds = Lists.newArrayList();
|
feeds = Lists.newArrayList();
|
||||||
@@ -143,27 +155,38 @@ public class FeedRefreshTaskGiver {
|
|||||||
feeds = feedDAO.findNextUpdatable(count, getThreshold());
|
feeds = feedDAO.findNextUpdatable(count, getThreshold());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// then, add to those the feeds we got from the add() method. We add them at the beginning of the list as they probably have a
|
||||||
|
// higher priority
|
||||||
int size = addQueue.size();
|
int size = addQueue.size();
|
||||||
for (int i = 0; i < size; i++) {
|
for (int i = 0; i < size; i++) {
|
||||||
feeds.add(0, addQueue.poll());
|
feeds.add(0, addQueue.poll());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// set the disabledDate to now as we use the disabledDate in feedDAO to decide what to refresh next. We also use a map to remove
|
||||||
|
// duplicates.
|
||||||
Map<Long, Feed> map = Maps.newLinkedHashMap();
|
Map<Long, Feed> map = Maps.newLinkedHashMap();
|
||||||
for (Feed f : feeds) {
|
for (Feed f : feeds) {
|
||||||
f.setDisabledUntil(new Date());
|
f.setDisabledUntil(new Date());
|
||||||
map.put(f.getId(), f);
|
map.put(f.getId(), f);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refill the queue
|
||||||
takeQueue.addAll(map.values());
|
takeQueue.addAll(map.values());
|
||||||
|
|
||||||
|
// add feeds from the giveBack queue to the map, overriding duplicates
|
||||||
size = giveBackQueue.size();
|
size = giveBackQueue.size();
|
||||||
for (int i = 0; i < size; i++) {
|
for (int i = 0; i < size; i++) {
|
||||||
Feed f = giveBackQueue.poll();
|
Feed f = giveBackQueue.poll();
|
||||||
map.put(f.getId(), f);
|
map.put(f.getId(), f);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// update all feeds in the database
|
||||||
feedDAO.saveOrUpdate(map.values());
|
feedDAO.saveOrUpdate(map.values());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* give a feed back, updating it to the database during the next refill()
|
||||||
|
*/
|
||||||
public void giveBack(Feed feed) {
|
public void giveBack(Feed feed) {
|
||||||
String normalized = FeedUtils.normalizeURL(feed.getUrl());
|
String normalized = FeedUtils.normalizeURL(feed.getUrl());
|
||||||
feed.setNormalizedUrl(normalized);
|
feed.setNormalizedUrl(normalized);
|
||||||
|
|||||||
Reference in New Issue
Block a user