Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-03-21 21:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

better refresh algorithm using queues instead of synchronous database call

Browse Source
This commit is contained in:
Athou
2013-06-30 12:18:24 +02:00
parent 0d2b9131d3
commit d3dd084dd7
8 changed files with 225 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java
View File

@@ -3,16 +3,13 @@ package com.commafeed.backend.feeds;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
@@ -24,6 +21,7 @@ import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feeds.FeedRefreshExecutor.Task;
import com.commafeed.backend.model.ApplicationSettings;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
@@ -34,7 +32,7 @@ import com.commafeed.backend.services.FeedUpdateService;
import com.google.api.client.util.Lists;
import com.google.common.util.concurrent.Striped;
@Singleton
@ApplicationScoped
public class FeedRefreshUpdater {
protected static Logger log = LoggerFactory
@@ -67,71 +65,33 @@ public class FeedRefreshUpdater {
@Inject
CacheService cache;
private ThreadPoolExecutor pool;
private FeedRefreshExecutor pool;
private Striped<Lock> locks;
private LinkedBlockingDeque<Runnable> queue;
@PostConstruct
public void init() {
ApplicationSettings settings = applicationSettingsService.get();
int threads = Math.max(settings.getDatabaseUpdateThreads(), 1);
log.info("Creating database pool with {} threads", threads);
pool = new FeedRefreshExecutor("FeedRefreshUpdater", threads, 500 * threads);
locks = Striped.lazyWeakLock(threads * 100000);
pool = new ThreadPoolExecutor(threads, threads, 0,
TimeUnit.MILLISECONDS,
queue = new LinkedBlockingDeque<Runnable>(500 * threads) {
private static final long serialVersionUID = 1L;
@Override
public boolean offer(Runnable r) {
Task task = (Task) r;
if (task.getFeed().isUrgent()) {
return offerFirst(r);
} else {
return offerLast(r);
}
}
});
pool.setRejectedExecutionHandler(new RejectedExecutionHandler() {
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
log.debug("Thread queue full, waiting...");
try {
Task task = (Task) r;
if (task.getFeed().isUrgent()) {
queue.putFirst(r);
} else {
queue.put(r);
}
} catch (InterruptedException e1) {
log.error("Interrupted while waiting for queue.", e1);
}
}
});
}
@PreDestroy
public void shutdown() {
pool.shutdownNow();
while (!pool.isTerminated()) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
log.error("interrupted while waiting for threads to finish.");
}
}
pool.shutdown();
}
public void updateFeed(Feed feed, Collection<FeedEntry> entries) {
pool.execute(new Task(feed, entries));
pool.execute(new EntryTask(feed, entries));
}
private class Task implements Runnable {
private class EntryTask implements Task {
private Feed feed;
private Collection<FeedEntry> entries;
public Task(Feed feed, Collection<FeedEntry> entries) {
public EntryTask(Feed feed, Collection<FeedEntry> entries) {
this.feed = feed;
this.entries = entries;
}
@@ -174,8 +134,9 @@ public class FeedRefreshUpdater {
taskGiver.giveBack(feed);
}
public Feed getFeed() {
return feed;
@Override
public boolean isUrgent() {
return feed.isUrgent();
}
}
@@ -221,7 +182,7 @@ public class FeedRefreshUpdater {
}
public int getQueueSize() {
return queue.size();
return pool.getQueueSize();
}
}