better background refresh

src/main/java/com/commafeed/backend/StartupBean.java

@@ -16,6 +16,7 @@ import com.commafeed.backend.dao.FeedCategoryService;
 import com.commafeed.backend.dao.FeedService;
 import com.commafeed.backend.dao.FeedSubscriptionService;
 import com.commafeed.backend.dao.UserService;
+import com.commafeed.backend.feeds.FeedRefreshWorker;
 import com.commafeed.backend.model.ApplicationSettings;
 import com.commafeed.backend.model.Feed;
 import com.commafeed.backend.model.FeedCategory;
@@ -49,69 +50,79 @@ public class StartupBean {
 	@Inject
 	ApplicationSettingsService applicationSettingsService;
 
+	@Inject
+	FeedRefreshWorker worker;
+
 	private long startupTime;
 
 	@PostConstruct
 	private void init() {
 		startupTime = Calendar.getInstance().getTimeInMillis();
 		if (userService.getCount() == 0) {
-			log.info("Populating database with default values");
-
-			applicationSettingsService.save(new ApplicationSettings());
-
-			User user = userService.register(ADMIN_NAME, "admin",
-					Arrays.asList(Role.ADMIN, Role.USER));
-			userService.register("test", "test", Arrays.asList(Role.USER));
-
-			Feed dilbert = new Feed(
-					"http://feed.dilbert.com/dilbert/daily_strip");
-			feedService.save(dilbert);
-
-			Feed engadget = new Feed("http://www.engadget.com/rss.xml");
-			feedService.save(engadget);
-
-			Feed frandroid = new Feed("http://feeds.feedburner.com/frandroid");
-			feedService.save(frandroid);
-
-			FeedCategory newsCategory = new FeedCategory();
-			newsCategory.setName("News");
-			newsCategory.setUser(user);
-			feedCategoryService.save(newsCategory);
-
-			FeedCategory comicsCategory = new FeedCategory();
-			comicsCategory.setName("Comics");
-			comicsCategory.setUser(user);
-			comicsCategory.setParent(newsCategory);
-			feedCategoryService.save(comicsCategory);
-
-			FeedCategory techCategory = new FeedCategory();
-			techCategory.setName("Tech");
-			techCategory.setUser(user);
-			techCategory.setParent(newsCategory);
-			feedCategoryService.save(techCategory);
-
-			FeedSubscription sub = new FeedSubscription();
-			sub.setCategory(comicsCategory);
-			sub.setFeed(dilbert);
-			sub.setTitle("Dilbert - Strips");
-			sub.setUser(user);
-			feedSubscriptionService.save(sub);
-
-			FeedSubscription sub2 = new FeedSubscription();
-			sub2.setCategory(techCategory);
-			sub2.setFeed(engadget);
-			sub2.setTitle("Engadget");
-			sub2.setUser(user);
-			feedSubscriptionService.save(sub2);
-
-			FeedSubscription sub3 = new FeedSubscription();
-			sub3.setFeed(frandroid);
-			sub3.setTitle("Frandroid");
-			sub3.setUser(user);
-			feedSubscriptionService.save(sub3);
-
+			initialData();
 		}
 
+		// 3 threads
+		for (int i = 0; i < 3; i++) {
+			worker.start();
+		}
+
+	}
+
+	private void initialData() {
+		log.info("Populating database with default values");
+
+		applicationSettingsService.save(new ApplicationSettings());
+
+		User user = userService.register(ADMIN_NAME, "admin",
+				Arrays.asList(Role.ADMIN, Role.USER));
+		userService.register("test", "test", Arrays.asList(Role.USER));
+
+		Feed dilbert = new Feed("http://feed.dilbert.com/dilbert/daily_strip");
+		feedService.save(dilbert);
+
+		Feed engadget = new Feed("http://www.engadget.com/rss.xml");
+		feedService.save(engadget);
+
+		Feed frandroid = new Feed("http://feeds.feedburner.com/frandroid");
+		feedService.save(frandroid);
+
+		FeedCategory newsCategory = new FeedCategory();
+		newsCategory.setName("News");
+		newsCategory.setUser(user);
+		feedCategoryService.save(newsCategory);
+
+		FeedCategory comicsCategory = new FeedCategory();
+		comicsCategory.setName("Comics");
+		comicsCategory.setUser(user);
+		comicsCategory.setParent(newsCategory);
+		feedCategoryService.save(comicsCategory);
+
+		FeedCategory techCategory = new FeedCategory();
+		techCategory.setName("Tech");
+		techCategory.setUser(user);
+		techCategory.setParent(newsCategory);
+		feedCategoryService.save(techCategory);
+
+		FeedSubscription sub = new FeedSubscription();
+		sub.setCategory(comicsCategory);
+		sub.setFeed(dilbert);
+		sub.setTitle("Dilbert - Strips");
+		sub.setUser(user);
+		feedSubscriptionService.save(sub);
+
+		FeedSubscription sub2 = new FeedSubscription();
+		sub2.setCategory(techCategory);
+		sub2.setFeed(engadget);
+		sub2.setTitle("Engadget");
+		sub2.setUser(user);
+		feedSubscriptionService.save(sub2);
+
+		FeedSubscription sub3 = new FeedSubscription();
+		sub3.setFeed(frandroid);
+		sub3.setTitle("Frandroid");
+		sub3.setUser(user);
+		feedSubscriptionService.save(sub3);
 	}
 
 	public long getStartupTime() {

src/main/java/com/commafeed/backend/dao/FeedService.java

@@ -1,10 +1,19 @@
 package com.commafeed.backend.dao;
 
+import java.util.Calendar;
+import java.util.Date;
 import java.util.List;
 
 import javax.ejb.Stateless;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.CriteriaQuery;
+import javax.persistence.criteria.Predicate;
+import javax.persistence.criteria.Root;
+
+import org.apache.commons.lang.time.DateUtils;
 
 import com.commafeed.backend.model.Feed;
+import com.commafeed.backend.model.Feed_;
 import com.commafeed.frontend.utils.ModelFactory.MF;
 import com.google.common.collect.Iterables;
 import com.uaihebert.model.EasyCriteria;
@@ -14,10 +23,32 @@ import com.uaihebert.model.EasyCriteria;
 public class FeedService extends GenericDAO<Feed> {
 
 	public List<Feed> findNextUpdatable(int count) {
-		EasyCriteria<Feed> criteria = createCriteria();
-		criteria.orderByAsc(MF.i(proxy().getLastUpdated()));
-		criteria.setMaxResults(count);
-		return criteria.getResultList();
+		CriteriaQuery<Feed> query = builder.createQuery(getType());
+		Root<Feed> root = query.from(getType());
+
+		Date now = Calendar.getInstance().getTime();
+
+		Predicate hasSubscriptions = builder.isNotEmpty(root
+				.get(Feed_.subscriptions));
+
+		Predicate neverUpdated = builder.isNull(root.get(Feed_.lastUpdated));
+		Predicate updatedMoreThanOneMinuteAgo = builder.lessThan(
+				root.get(Feed_.lastUpdated), DateUtils.addMinutes(now, -1));
+
+		Predicate disabledDateIsNull = builder.isNull(root
+				.get(Feed_.disabledUntil));
+		Predicate DisabledDateIsInPast = builder.lessThan(
+				root.get(Feed_.disabledUntil), now);
+
+		query.where(hasSubscriptions,
+				builder.or(neverUpdated, updatedMoreThanOneMinuteAgo),
+				builder.or(disabledDateIsNull, DisabledDateIsInPast));
+		query.orderBy(builder.asc(root.get(Feed_.lastUpdated)));
+
+		TypedQuery<Feed> q = em.createQuery(query);
+		q.setMaxResults(count);
+
+		return q.getResultList();
 	}
 
 	public Feed findByUrl(String url) {
@@ -29,7 +60,7 @@ public class FeedService extends GenericDAO<Feed> {
 		EasyCriteria<Feed> criteria = createCriteria();
 		criteria.andEquals(MF.i(proxy().getId()), feedId);
 		criteria.leftJoinFetch(MF.i(proxy().getEntries()));
-		
+
 		criteria.setFirstResult(offset);
 		criteria.setMaxResults(limit);
 		return criteria.getSingleResult();

src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java

@@ -0,0 +1,136 @@
+package com.commafeed.backend.feeds;
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.concurrent.locks.ReentrantLock;
+
+import javax.annotation.Resource;
+import javax.ejb.Asynchronous;
+import javax.ejb.EJBException;
+import javax.ejb.Stateless;
+import javax.ejb.TransactionManagement;
+import javax.ejb.TransactionManagementType;
+import javax.inject.Inject;
+import javax.transaction.HeuristicMixedException;
+import javax.transaction.HeuristicRollbackException;
+import javax.transaction.NotSupportedException;
+import javax.transaction.RollbackException;
+import javax.transaction.SystemException;
+import javax.transaction.UserTransaction;
+
+import org.apache.commons.lang.time.DateUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.commafeed.backend.dao.FeedEntryService;
+import com.commafeed.backend.dao.FeedService;
+import com.commafeed.backend.model.Feed;
+import com.google.common.collect.Iterables;
+
+@Stateless
+@TransactionManagement(TransactionManagementType.BEAN)
+public class FeedRefreshWorker {
+
+	private static Logger log = LoggerFactory
+			.getLogger(FeedRefreshWorker.class);
+
+	private static final ReentrantLock lock = new ReentrantLock();
+
+	@Inject
+	FeedFetcher fetcher;
+
+	@Inject
+	FeedService feedService;
+
+	@Inject
+	FeedEntryService feedEntryService;
+
+	@Resource
+	private UserTransaction transaction;
+
+	@Asynchronous
+	public void start() {
+		while (true) {
+			try {
+				Feed feed = getNextFeed();
+				if (feed != null) {
+					System.out.println("refreshing " + feed.getUrl());
+					update(feed);
+				} else {
+					System.out.println("sleeping");
+					Thread.sleep(15000);
+				}
+			} catch (Exception e) {
+				throw new EJBException(e.getMessage(), e);
+			}
+		}
+	}
+
+	private void update(Feed feed) throws NotSupportedException,
+			SystemException, SecurityException, IllegalStateException,
+			RollbackException, HeuristicMixedException,
+			HeuristicRollbackException {
+
+		String message = null;
+		int errorCount = 0;
+		Date disabledUntil = null;
+
+		Feed fetchedFeed = null;
+		try {
+			fetchedFeed = fetcher.fetch(feed.getUrl());
+		} catch (Exception e) {
+			e.printStackTrace();
+			message = "Unable to refresh feed " + feed.getUrl() + " : "
+					+ e.getMessage();
+			log.info(e.getClass().getName() + " " + message);
+
+			errorCount = feed.getErrorCount() + 1;
+
+			int retriesBeforeDisable = 3;
+			if (feed.getErrorCount() >= retriesBeforeDisable) {
+				int disabledMinutes = 10 * (feed.getErrorCount()
+						- retriesBeforeDisable + 1);
+				disabledMinutes = Math.min(60, disabledMinutes);
+				disabledUntil = DateUtils.addMinutes(Calendar.getInstance()
+						.getTime(), disabledMinutes);
+			}
+		}
+
+		feed.setMessage(message);
+		feed.setErrorCount(errorCount);
+		feed.setDisabledUntil(disabledUntil);
+
+		transaction.begin();
+
+		if (fetchedFeed != null) {
+			feedEntryService.updateEntries(feed.getUrl(),
+					fetchedFeed.getEntries());
+			if (feed.getLink() == null) {
+				feed.setLink(fetchedFeed.getLink());
+			}
+		}
+		feedService.update(feed);
+
+		transaction.commit();
+
+	}
+
+	private Feed getNextFeed() throws NotSupportedException, SystemException,
+			SecurityException, IllegalStateException, RollbackException,
+			HeuristicMixedException, HeuristicRollbackException {
+
+		Feed feed = null;
+		lock.lock();
+		try {
+			feed = Iterables.getFirst(feedService.findNextUpdatable(1), null);
+			if (feed != null) {
+				feed.setLastUpdated(Calendar.getInstance().getTime());
+				feedService.update(feed);
+			}
+		} finally {
+			lock.unlock();
+		}
+
+		return feed;
+	}
+}

src/main/java/com/commafeed/backend/feeds/FeedTimer.java

@@ -1,45 +0,0 @@
-package com.commafeed.backend.feeds;
-
-import java.util.List;
-
-import javax.ejb.Lock;
-import javax.ejb.LockType;
-import javax.ejb.Schedule;
-import javax.ejb.Singleton;
-import javax.inject.Inject;
-
-import com.commafeed.backend.dao.FeedService;
-import com.commafeed.backend.model.Feed;
-
-@Singleton
-public class FeedTimer {
-
-	@Inject
-	FeedService feedService;
-
-	@Inject
-	FeedUpdater updater;
-
-	private long count = -1;
-
-	// every seconds
-	@Schedule(hour = "*", minute = "*", second = "*/5", persistent = false)
-	@Lock(LockType.READ)
-	private void timeout() {
-		if (count == -1) {
-			refreshCount();
-		}
-		if (count > 0) {
-			int updateCount = (int) Math.ceil(count * 5d / 60d);
-			List<Feed> feeds = feedService.findNextUpdatable(updateCount);
-			for (Feed feed : feeds) {
-				updater.update(feed);
-			}
-		}
-	}
-
-	@Schedule(hour = "*", minute = "0", persistent = false)
-	private void refreshCount() {
-		count = feedService.getCount();
-	}
-}

src/main/java/com/commafeed/backend/feeds/FeedUpdater.java

@@ -1,80 +0,0 @@
-package com.commafeed.backend.feeds;
-
-import java.util.Calendar;
-import java.util.Date;
-import java.util.concurrent.TimeUnit;
-
-import javax.ejb.AccessTimeout;
-import javax.ejb.Asynchronous;
-import javax.ejb.Lock;
-import javax.ejb.LockType;
-import javax.ejb.Stateless;
-import javax.inject.Inject;
-
-import org.apache.commons.lang.time.DateUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.commafeed.backend.dao.FeedEntryService;
-import com.commafeed.backend.dao.FeedService;
-import com.commafeed.backend.model.Feed;
-
-@Stateless
-public class FeedUpdater {
-
-	private static Logger log = LoggerFactory.getLogger(FeedTimer.class);
-
-	@Inject
-	FeedFetcher fetcher;
-
-	@Inject
-	FeedService feedService;
-
-	@Inject
-	FeedEntryService feedEntryService;
-
-	@Asynchronous
-	@Lock(LockType.READ)
-	@AccessTimeout(value = 4, unit = TimeUnit.SECONDS)
-	public void update(Feed feed) {
-
-		try {
-
-			Date now = Calendar.getInstance().getTime();
-			Date disabledUntil = feed.getDisabledUntil();
-
-			if (disabledUntil == null || disabledUntil.before(now)) {
-				Feed fetchedFeed = fetcher.fetch(feed.getUrl());
-				if (feed.getLink() == null) {
-					feed.setLink(fetchedFeed.getLink());
-					feedService.update(feed);
-				}
-				feedEntryService.updateEntries(feed.getUrl(),
-						fetchedFeed.getEntries());
-
-				feed.setMessage(null);
-				feed.setErrorCount(0);
-				feed.setDisabledUntil(null);
-			}
-		} catch (Exception e) {
-			String message = "Unable to refresh feed " + feed.getUrl() + " : "
-					+ e.getMessage();
-			log.info(e.getClass().getName() + " " + message);
-			feed.setMessage(message);
-			feed.setErrorCount(feed.getErrorCount() + 1);
-
-			int retriesBeforeDisable = 3;
-
-			if (feed.getErrorCount() >= retriesBeforeDisable) {
-				int disabledMinutes = 10 * (feed.getErrorCount()
-						- retriesBeforeDisable + 1);
-				disabledMinutes = Math.min(60, disabledMinutes);
-				feed.setDisabledUntil(DateUtils.addMinutes(Calendar
-						.getInstance().getTime(), disabledMinutes));
-			}
-		} finally {
-			feed.setLastUpdated(Calendar.getInstance().getTime());
-			feedService.update(feed);
-		}
-	}
-}