From 4b49625a8fb6cf50bbf8ade42755eab6ce8b2721 Mon Sep 17 00:00:00 2001 From: Athou Date: Thu, 11 Apr 2013 19:11:27 +0200 Subject: [PATCH] better background refresh --- .../com/commafeed/backend/StartupBean.java | 121 +++++++++------- .../commafeed/backend/dao/FeedService.java | 41 +++++- .../backend/feeds/FeedRefreshWorker.java | 136 ++++++++++++++++++ .../commafeed/backend/feeds/FeedTimer.java | 45 ------ .../commafeed/backend/feeds/FeedUpdater.java | 80 ----------- 5 files changed, 238 insertions(+), 185 deletions(-) create mode 100644 src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java delete mode 100644 src/main/java/com/commafeed/backend/feeds/FeedTimer.java delete mode 100644 src/main/java/com/commafeed/backend/feeds/FeedUpdater.java diff --git a/src/main/java/com/commafeed/backend/StartupBean.java b/src/main/java/com/commafeed/backend/StartupBean.java index d032485d..be93c620 100644 --- a/src/main/java/com/commafeed/backend/StartupBean.java +++ b/src/main/java/com/commafeed/backend/StartupBean.java @@ -16,6 +16,7 @@ import com.commafeed.backend.dao.FeedCategoryService; import com.commafeed.backend.dao.FeedService; import com.commafeed.backend.dao.FeedSubscriptionService; import com.commafeed.backend.dao.UserService; +import com.commafeed.backend.feeds.FeedRefreshWorker; import com.commafeed.backend.model.ApplicationSettings; import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedCategory; @@ -49,69 +50,79 @@ public class StartupBean { @Inject ApplicationSettingsService applicationSettingsService; + @Inject + FeedRefreshWorker worker; + private long startupTime; @PostConstruct private void init() { startupTime = Calendar.getInstance().getTimeInMillis(); if (userService.getCount() == 0) { - log.info("Populating database with default values"); - - applicationSettingsService.save(new ApplicationSettings()); - - User user = userService.register(ADMIN_NAME, "admin", - Arrays.asList(Role.ADMIN, Role.USER)); - userService.register("test", "test", Arrays.asList(Role.USER)); - - Feed dilbert = new Feed( - "http://feed.dilbert.com/dilbert/daily_strip"); - feedService.save(dilbert); - - Feed engadget = new Feed("http://www.engadget.com/rss.xml"); - feedService.save(engadget); - - Feed frandroid = new Feed("http://feeds.feedburner.com/frandroid"); - feedService.save(frandroid); - - FeedCategory newsCategory = new FeedCategory(); - newsCategory.setName("News"); - newsCategory.setUser(user); - feedCategoryService.save(newsCategory); - - FeedCategory comicsCategory = new FeedCategory(); - comicsCategory.setName("Comics"); - comicsCategory.setUser(user); - comicsCategory.setParent(newsCategory); - feedCategoryService.save(comicsCategory); - - FeedCategory techCategory = new FeedCategory(); - techCategory.setName("Tech"); - techCategory.setUser(user); - techCategory.setParent(newsCategory); - feedCategoryService.save(techCategory); - - FeedSubscription sub = new FeedSubscription(); - sub.setCategory(comicsCategory); - sub.setFeed(dilbert); - sub.setTitle("Dilbert - Strips"); - sub.setUser(user); - feedSubscriptionService.save(sub); - - FeedSubscription sub2 = new FeedSubscription(); - sub2.setCategory(techCategory); - sub2.setFeed(engadget); - sub2.setTitle("Engadget"); - sub2.setUser(user); - feedSubscriptionService.save(sub2); - - FeedSubscription sub3 = new FeedSubscription(); - sub3.setFeed(frandroid); - sub3.setTitle("Frandroid"); - sub3.setUser(user); - feedSubscriptionService.save(sub3); - + initialData(); } + // 3 threads + for (int i = 0; i < 3; i++) { + worker.start(); + } + + } + + private void initialData() { + log.info("Populating database with default values"); + + applicationSettingsService.save(new ApplicationSettings()); + + User user = userService.register(ADMIN_NAME, "admin", + Arrays.asList(Role.ADMIN, Role.USER)); + userService.register("test", "test", Arrays.asList(Role.USER)); + + Feed dilbert = new Feed("http://feed.dilbert.com/dilbert/daily_strip"); + feedService.save(dilbert); + + Feed engadget = new Feed("http://www.engadget.com/rss.xml"); + feedService.save(engadget); + + Feed frandroid = new Feed("http://feeds.feedburner.com/frandroid"); + feedService.save(frandroid); + + FeedCategory newsCategory = new FeedCategory(); + newsCategory.setName("News"); + newsCategory.setUser(user); + feedCategoryService.save(newsCategory); + + FeedCategory comicsCategory = new FeedCategory(); + comicsCategory.setName("Comics"); + comicsCategory.setUser(user); + comicsCategory.setParent(newsCategory); + feedCategoryService.save(comicsCategory); + + FeedCategory techCategory = new FeedCategory(); + techCategory.setName("Tech"); + techCategory.setUser(user); + techCategory.setParent(newsCategory); + feedCategoryService.save(techCategory); + + FeedSubscription sub = new FeedSubscription(); + sub.setCategory(comicsCategory); + sub.setFeed(dilbert); + sub.setTitle("Dilbert - Strips"); + sub.setUser(user); + feedSubscriptionService.save(sub); + + FeedSubscription sub2 = new FeedSubscription(); + sub2.setCategory(techCategory); + sub2.setFeed(engadget); + sub2.setTitle("Engadget"); + sub2.setUser(user); + feedSubscriptionService.save(sub2); + + FeedSubscription sub3 = new FeedSubscription(); + sub3.setFeed(frandroid); + sub3.setTitle("Frandroid"); + sub3.setUser(user); + feedSubscriptionService.save(sub3); } public long getStartupTime() { diff --git a/src/main/java/com/commafeed/backend/dao/FeedService.java b/src/main/java/com/commafeed/backend/dao/FeedService.java index 876469ad..77b949b2 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedService.java +++ b/src/main/java/com/commafeed/backend/dao/FeedService.java @@ -1,10 +1,19 @@ package com.commafeed.backend.dao; +import java.util.Calendar; +import java.util.Date; import java.util.List; import javax.ejb.Stateless; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; + +import org.apache.commons.lang.time.DateUtils; import com.commafeed.backend.model.Feed; +import com.commafeed.backend.model.Feed_; import com.commafeed.frontend.utils.ModelFactory.MF; import com.google.common.collect.Iterables; import com.uaihebert.model.EasyCriteria; @@ -14,10 +23,32 @@ import com.uaihebert.model.EasyCriteria; public class FeedService extends GenericDAO { public List findNextUpdatable(int count) { - EasyCriteria criteria = createCriteria(); - criteria.orderByAsc(MF.i(proxy().getLastUpdated())); - criteria.setMaxResults(count); - return criteria.getResultList(); + CriteriaQuery query = builder.createQuery(getType()); + Root root = query.from(getType()); + + Date now = Calendar.getInstance().getTime(); + + Predicate hasSubscriptions = builder.isNotEmpty(root + .get(Feed_.subscriptions)); + + Predicate neverUpdated = builder.isNull(root.get(Feed_.lastUpdated)); + Predicate updatedMoreThanOneMinuteAgo = builder.lessThan( + root.get(Feed_.lastUpdated), DateUtils.addMinutes(now, -1)); + + Predicate disabledDateIsNull = builder.isNull(root + .get(Feed_.disabledUntil)); + Predicate DisabledDateIsInPast = builder.lessThan( + root.get(Feed_.disabledUntil), now); + + query.where(hasSubscriptions, + builder.or(neverUpdated, updatedMoreThanOneMinuteAgo), + builder.or(disabledDateIsNull, DisabledDateIsInPast)); + query.orderBy(builder.asc(root.get(Feed_.lastUpdated))); + + TypedQuery q = em.createQuery(query); + q.setMaxResults(count); + + return q.getResultList(); } public Feed findByUrl(String url) { @@ -29,7 +60,7 @@ public class FeedService extends GenericDAO { EasyCriteria criteria = createCriteria(); criteria.andEquals(MF.i(proxy().getId()), feedId); criteria.leftJoinFetch(MF.i(proxy().getEntries())); - + criteria.setFirstResult(offset); criteria.setMaxResults(limit); return criteria.getSingleResult(); diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java new file mode 100644 index 00000000..c4be9291 --- /dev/null +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java @@ -0,0 +1,136 @@ +package com.commafeed.backend.feeds; + +import java.util.Calendar; +import java.util.Date; +import java.util.concurrent.locks.ReentrantLock; + +import javax.annotation.Resource; +import javax.ejb.Asynchronous; +import javax.ejb.EJBException; +import javax.ejb.Stateless; +import javax.ejb.TransactionManagement; +import javax.ejb.TransactionManagementType; +import javax.inject.Inject; +import javax.transaction.HeuristicMixedException; +import javax.transaction.HeuristicRollbackException; +import javax.transaction.NotSupportedException; +import javax.transaction.RollbackException; +import javax.transaction.SystemException; +import javax.transaction.UserTransaction; + +import org.apache.commons.lang.time.DateUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.commafeed.backend.dao.FeedEntryService; +import com.commafeed.backend.dao.FeedService; +import com.commafeed.backend.model.Feed; +import com.google.common.collect.Iterables; + +@Stateless +@TransactionManagement(TransactionManagementType.BEAN) +public class FeedRefreshWorker { + + private static Logger log = LoggerFactory + .getLogger(FeedRefreshWorker.class); + + private static final ReentrantLock lock = new ReentrantLock(); + + @Inject + FeedFetcher fetcher; + + @Inject + FeedService feedService; + + @Inject + FeedEntryService feedEntryService; + + @Resource + private UserTransaction transaction; + + @Asynchronous + public void start() { + while (true) { + try { + Feed feed = getNextFeed(); + if (feed != null) { + System.out.println("refreshing " + feed.getUrl()); + update(feed); + } else { + System.out.println("sleeping"); + Thread.sleep(15000); + } + } catch (Exception e) { + throw new EJBException(e.getMessage(), e); + } + } + } + + private void update(Feed feed) throws NotSupportedException, + SystemException, SecurityException, IllegalStateException, + RollbackException, HeuristicMixedException, + HeuristicRollbackException { + + String message = null; + int errorCount = 0; + Date disabledUntil = null; + + Feed fetchedFeed = null; + try { + fetchedFeed = fetcher.fetch(feed.getUrl()); + } catch (Exception e) { + e.printStackTrace(); + message = "Unable to refresh feed " + feed.getUrl() + " : " + + e.getMessage(); + log.info(e.getClass().getName() + " " + message); + + errorCount = feed.getErrorCount() + 1; + + int retriesBeforeDisable = 3; + if (feed.getErrorCount() >= retriesBeforeDisable) { + int disabledMinutes = 10 * (feed.getErrorCount() + - retriesBeforeDisable + 1); + disabledMinutes = Math.min(60, disabledMinutes); + disabledUntil = DateUtils.addMinutes(Calendar.getInstance() + .getTime(), disabledMinutes); + } + } + + feed.setMessage(message); + feed.setErrorCount(errorCount); + feed.setDisabledUntil(disabledUntil); + + transaction.begin(); + + if (fetchedFeed != null) { + feedEntryService.updateEntries(feed.getUrl(), + fetchedFeed.getEntries()); + if (feed.getLink() == null) { + feed.setLink(fetchedFeed.getLink()); + } + } + feedService.update(feed); + + transaction.commit(); + + } + + private Feed getNextFeed() throws NotSupportedException, SystemException, + SecurityException, IllegalStateException, RollbackException, + HeuristicMixedException, HeuristicRollbackException { + + Feed feed = null; + lock.lock(); + try { + feed = Iterables.getFirst(feedService.findNextUpdatable(1), null); + if (feed != null) { + feed.setLastUpdated(Calendar.getInstance().getTime()); + feedService.update(feed); + } + } finally { + lock.unlock(); + } + + return feed; + } +} diff --git a/src/main/java/com/commafeed/backend/feeds/FeedTimer.java b/src/main/java/com/commafeed/backend/feeds/FeedTimer.java deleted file mode 100644 index 0dbb9908..00000000 --- a/src/main/java/com/commafeed/backend/feeds/FeedTimer.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.commafeed.backend.feeds; - -import java.util.List; - -import javax.ejb.Lock; -import javax.ejb.LockType; -import javax.ejb.Schedule; -import javax.ejb.Singleton; -import javax.inject.Inject; - -import com.commafeed.backend.dao.FeedService; -import com.commafeed.backend.model.Feed; - -@Singleton -public class FeedTimer { - - @Inject - FeedService feedService; - - @Inject - FeedUpdater updater; - - private long count = -1; - - // every seconds - @Schedule(hour = "*", minute = "*", second = "*/5", persistent = false) - @Lock(LockType.READ) - private void timeout() { - if (count == -1) { - refreshCount(); - } - if (count > 0) { - int updateCount = (int) Math.ceil(count * 5d / 60d); - List feeds = feedService.findNextUpdatable(updateCount); - for (Feed feed : feeds) { - updater.update(feed); - } - } - } - - @Schedule(hour = "*", minute = "0", persistent = false) - private void refreshCount() { - count = feedService.getCount(); - } -} diff --git a/src/main/java/com/commafeed/backend/feeds/FeedUpdater.java b/src/main/java/com/commafeed/backend/feeds/FeedUpdater.java deleted file mode 100644 index ea673e79..00000000 --- a/src/main/java/com/commafeed/backend/feeds/FeedUpdater.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.commafeed.backend.feeds; - -import java.util.Calendar; -import java.util.Date; -import java.util.concurrent.TimeUnit; - -import javax.ejb.AccessTimeout; -import javax.ejb.Asynchronous; -import javax.ejb.Lock; -import javax.ejb.LockType; -import javax.ejb.Stateless; -import javax.inject.Inject; - -import org.apache.commons.lang.time.DateUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.commafeed.backend.dao.FeedEntryService; -import com.commafeed.backend.dao.FeedService; -import com.commafeed.backend.model.Feed; - -@Stateless -public class FeedUpdater { - - private static Logger log = LoggerFactory.getLogger(FeedTimer.class); - - @Inject - FeedFetcher fetcher; - - @Inject - FeedService feedService; - - @Inject - FeedEntryService feedEntryService; - - @Asynchronous - @Lock(LockType.READ) - @AccessTimeout(value = 4, unit = TimeUnit.SECONDS) - public void update(Feed feed) { - - try { - - Date now = Calendar.getInstance().getTime(); - Date disabledUntil = feed.getDisabledUntil(); - - if (disabledUntil == null || disabledUntil.before(now)) { - Feed fetchedFeed = fetcher.fetch(feed.getUrl()); - if (feed.getLink() == null) { - feed.setLink(fetchedFeed.getLink()); - feedService.update(feed); - } - feedEntryService.updateEntries(feed.getUrl(), - fetchedFeed.getEntries()); - - feed.setMessage(null); - feed.setErrorCount(0); - feed.setDisabledUntil(null); - } - } catch (Exception e) { - String message = "Unable to refresh feed " + feed.getUrl() + " : " - + e.getMessage(); - log.info(e.getClass().getName() + " " + message); - feed.setMessage(message); - feed.setErrorCount(feed.getErrorCount() + 1); - - int retriesBeforeDisable = 3; - - if (feed.getErrorCount() >= retriesBeforeDisable) { - int disabledMinutes = 10 * (feed.getErrorCount() - - retriesBeforeDisable + 1); - disabledMinutes = Math.min(60, disabledMinutes); - feed.setDisabledUntil(DateUtils.addMinutes(Calendar - .getInstance().getTime(), disabledMinutes)); - } - } finally { - feed.setLastUpdated(Calendar.getInstance().getTime()); - feedService.update(feed); - } - } -}