Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-03-21 21:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

better refresh algorithm using queues instead of synchronous database call

Browse Source
This commit is contained in:
Athou
2013-06-30 12:18:24 +02:00
parent 0d2b9131d3
commit d3dd084dd7
8 changed files with 225 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java
View File

@@ -4,12 +4,15 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -20,7 +23,7 @@ import com.commafeed.backend.services.ApplicationSettingsService;
import com.google.api.client.util.Maps;
import com.google.common.collect.Queues;
@Singleton
@ApplicationScoped
public class FeedRefreshTaskGiver {
protected static final Logger log = LoggerFactory.getLogger(FeedRefreshTaskGiver.class);
@@ -34,47 +37,103 @@ public class FeedRefreshTaskGiver {
@Inject
MetricsBean metricsBean;
@Inject
FeedRefreshWorker worker;
private int backgroundThreads;
private Queue<Feed> addQueue = Queues.newConcurrentLinkedQueue();
private Queue<Feed> takeQueue = Queues.newConcurrentLinkedQueue();
private Queue<Feed> giveBackQueue = Queues.newConcurrentLinkedQueue();
private ExecutorService executor;
@PostConstruct
public void init() {
backgroundThreads = applicationSettingsService.get()
.getBackgroundThreads();
executor = Executors.newFixedThreadPool(1);
}
public void add(Feed feed) {
Date now = new Date();
boolean heavyLoad = applicationSettingsService.get().isHeavyLoad();
Date threshold = DateUtils.addMinutes(now, heavyLoad ? -10 : -1);
if (feed.getLastUpdated() == null
|| feed.getLastUpdated().before(threshold)) {
addQueue.add(feed);
@PreDestroy
public void shutdown() {
executor.shutdownNow();
while (!executor.isTerminated()) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
log.error("interrupted while waiting for threads to finish.");
}
}
}
public synchronized Feed take() {
public void start() {
try {
// sleeping for a little while, let everything settle
Thread.sleep(5000);
} catch (InterruptedException e) {
log.error("interrupted while sleeping");
}
log.info("starting feed refresh task giver");
executor.execute(new Runnable() {
@Override
public void run() {
while (!executor.isShutdown()) {
try {
Feed feed = take();
if (feed != null) {
metricsBean.feedRefreshed();
worker.updateFeed(feed);
} else {
log.debug("nothing to do, sleeping for 15s");
try {
Thread.sleep(15000);
} catch (InterruptedException e) {
log.error("interrupted while sleeping");
}
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
}
});
}
private Feed take() {
Feed feed = takeQueue.poll();
if (feed == null) {
refill();
feed = takeQueue.poll();
}
if (feed != null) {
metricsBean.feedRefreshed();
}
return feed;
}
public Long getUpdatableCount() {
return feedDAO.getUpdatableCount(getThreshold());
}
private Date getThreshold() {
boolean heavyLoad = applicationSettingsService.get().isHeavyLoad();
Date threshold = DateUtils.addMinutes(new Date(), heavyLoad ? -15 : -5);
return threshold;
}
public void add(Feed feed) {
Date threshold = getThreshold();
if (feed.getLastUpdated() == null
|| feed.getLastUpdated().before(threshold)) {
addQueue.add(feed);
}
}
private void refill() {
Date now = new Date();
int count = 3 * backgroundThreads;
List<Feed> feeds = feedDAO.findNextUpdatable(count);
List<Feed> feeds = feedDAO.findNextUpdatable(count, getThreshold());
int size = addQueue.size();
for (int i = 0; i < size; i++) {