Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-03-21 21:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

better background refresh

Browse Source
This commit is contained in:
Athou
2013-04-11 19:11:27 +02:00
parent 70b6fd1fd3
commit 4b49625a8f
5 changed files with 238 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java Normal file
View File

@@ -0,0 +1,136 @@
package com.commafeed.backend.feeds;
import java.util.Calendar;
import java.util.Date;
import java.util.concurrent.locks.ReentrantLock;
import javax.annotation.Resource;
import javax.ejb.Asynchronous;
import javax.ejb.EJBException;
import javax.ejb.Stateless;
import javax.ejb.TransactionManagement;
import javax.ejb.TransactionManagementType;
import javax.inject.Inject;
import javax.transaction.HeuristicMixedException;
import javax.transaction.HeuristicRollbackException;
import javax.transaction.NotSupportedException;
import javax.transaction.RollbackException;
import javax.transaction.SystemException;
import javax.transaction.UserTransaction;
import org.apache.commons.lang.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.commafeed.backend.dao.FeedEntryService;
import com.commafeed.backend.dao.FeedService;
import com.commafeed.backend.model.Feed;
import com.google.common.collect.Iterables;
@Stateless
@TransactionManagement(TransactionManagementType.BEAN)
public class FeedRefreshWorker {
private static Logger log = LoggerFactory
.getLogger(FeedRefreshWorker.class);
private static final ReentrantLock lock = new ReentrantLock();
@Inject
FeedFetcher fetcher;
@Inject
FeedService feedService;
@Inject
FeedEntryService feedEntryService;
@Resource
private UserTransaction transaction;
@Asynchronous
public void start() {
while (true) {
try {
Feed feed = getNextFeed();
if (feed != null) {
System.out.println("refreshing " + feed.getUrl());
update(feed);
} else {
System.out.println("sleeping");
Thread.sleep(15000);
}
} catch (Exception e) {
throw new EJBException(e.getMessage(), e);
}
}
}
private void update(Feed feed) throws NotSupportedException,
SystemException, SecurityException, IllegalStateException,
RollbackException, HeuristicMixedException,
HeuristicRollbackException {
String message = null;
int errorCount = 0;
Date disabledUntil = null;
Feed fetchedFeed = null;
try {
fetchedFeed = fetcher.fetch(feed.getUrl());
} catch (Exception e) {
e.printStackTrace();
message = "Unable to refresh feed " + feed.getUrl() + " : "
+ e.getMessage();
log.info(e.getClass().getName() + " " + message);
errorCount = feed.getErrorCount() + 1;
int retriesBeforeDisable = 3;
if (feed.getErrorCount() >= retriesBeforeDisable) {
int disabledMinutes = 10 * (feed.getErrorCount()
- retriesBeforeDisable + 1);
disabledMinutes = Math.min(60, disabledMinutes);
disabledUntil = DateUtils.addMinutes(Calendar.getInstance()
.getTime(), disabledMinutes);
}
}
feed.setMessage(message);
feed.setErrorCount(errorCount);
feed.setDisabledUntil(disabledUntil);
transaction.begin();
if (fetchedFeed != null) {
feedEntryService.updateEntries(feed.getUrl(),
fetchedFeed.getEntries());
if (feed.getLink() == null) {
feed.setLink(fetchedFeed.getLink());
}
}
feedService.update(feed);
transaction.commit();
}
private Feed getNextFeed() throws NotSupportedException, SystemException,
SecurityException, IllegalStateException, RollbackException,
HeuristicMixedException, HeuristicRollbackException {
Feed feed = null;
lock.lock();
try {
feed = Iterables.getFirst(feedService.findNextUpdatable(1), null);
if (feed != null) {
feed.setLastUpdated(Calendar.getInstance().getTime());
feedService.update(feed);
}
} finally {
lock.unlock();
}
return feed;
}
}

45
src/main/java/com/commafeed/backend/feeds/FeedTimer.java
View File

@@ -1,45 +0,0 @@
package com.commafeed.backend.feeds;
import java.util.List;
import javax.ejb.Lock;
import javax.ejb.LockType;
import javax.ejb.Schedule;
import javax.ejb.Singleton;
import javax.inject.Inject;
import com.commafeed.backend.dao.FeedService;
import com.commafeed.backend.model.Feed;
@Singleton
public class FeedTimer {
@Inject
FeedService feedService;
@Inject
FeedUpdater updater;
private long count = -1;
// every seconds
@Schedule(hour = "*", minute = "*", second = "*/5", persistent = false)
@Lock(LockType.READ)
private void timeout() {
if (count == -1) {
refreshCount();
}
if (count > 0) {
int updateCount = (int) Math.ceil(count * 5d / 60d);
List<Feed> feeds = feedService.findNextUpdatable(updateCount);
for (Feed feed : feeds) {
updater.update(feed);
}
}
}
@Schedule(hour = "*", minute = "0", persistent = false)
private void refreshCount() {
count = feedService.getCount();
}
}

80
src/main/java/com/commafeed/backend/feeds/FeedUpdater.java
View File

@@ -1,80 +0,0 @@
package com.commafeed.backend.feeds;
import java.util.Calendar;
import java.util.Date;
import java.util.concurrent.TimeUnit;
import javax.ejb.AccessTimeout;
import javax.ejb.Asynchronous;
import javax.ejb.Lock;
import javax.ejb.LockType;
import javax.ejb.Stateless;
import javax.inject.Inject;
import org.apache.commons.lang.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.commafeed.backend.dao.FeedEntryService;
import com.commafeed.backend.dao.FeedService;
import com.commafeed.backend.model.Feed;
@Stateless
public class FeedUpdater {
private static Logger log = LoggerFactory.getLogger(FeedTimer.class);
@Inject
FeedFetcher fetcher;
@Inject
FeedService feedService;
@Inject
FeedEntryService feedEntryService;
@Asynchronous
@Lock(LockType.READ)
@AccessTimeout(value = 4, unit = TimeUnit.SECONDS)
public void update(Feed feed) {
try {
Date now = Calendar.getInstance().getTime();
Date disabledUntil = feed.getDisabledUntil();
if (disabledUntil == null || disabledUntil.before(now)) {
Feed fetchedFeed = fetcher.fetch(feed.getUrl());
if (feed.getLink() == null) {
feed.setLink(fetchedFeed.getLink());
feedService.update(feed);
}
feedEntryService.updateEntries(feed.getUrl(),
fetchedFeed.getEntries());
feed.setMessage(null);
feed.setErrorCount(0);
feed.setDisabledUntil(null);
}
} catch (Exception e) {
String message = "Unable to refresh feed " + feed.getUrl() + " : "
+ e.getMessage();
log.info(e.getClass().getName() + " " + message);
feed.setMessage(message);
feed.setErrorCount(feed.getErrorCount() + 1);
int retriesBeforeDisable = 3;
if (feed.getErrorCount() >= retriesBeforeDisable) {
int disabledMinutes = 10 * (feed.getErrorCount()
- retriesBeforeDisable + 1);
disabledMinutes = Math.min(60, disabledMinutes);
feed.setDisabledUntil(DateUtils.addMinutes(Calendar
.getInstance().getTime(), disabledMinutes));
}
} finally {
feed.setLastUpdated(Calendar.getInstance().getTime());
feedService.update(feed);
}
}
}