From 0d83bd19d085e80d1becf34607e434eda372317c Mon Sep 17 00:00:00 2001 From: Jeremie Panzer Date: Wed, 27 Mar 2013 09:37:15 +0100 Subject: [PATCH] tweak feed fetching --- pom.xml | 8 +-- .../commafeed/backend/feeds/FeedFetcher.java | 26 ++++------ .../commafeed/backend/feeds/FeedParser.java | 7 --- .../commafeed/backend/feeds/FeedTimer.java | 44 +--------------- .../commafeed/backend/feeds/FeedUpdater.java | 51 +++++++++++++++++++ .../commafeed/backend/feeds/OPMLImporter.java | 4 +- 6 files changed, 69 insertions(+), 71 deletions(-) create mode 100644 src/main/java/com/commafeed/backend/feeds/FeedUpdater.java diff --git a/pom.xml b/pom.xml index f6b8fddc..a8b8b192 100644 --- a/pom.xml +++ b/pom.xml @@ -210,7 +210,7 @@ apache.releases - https://repository.apache.org/content/groups/public/ + http://repository.apache.org/content/groups/public/ true @@ -220,7 +220,7 @@ apache.snapshots - https://repository.apache.org/content/groups/snapshots/ + http://repository.apache.org/content/groups/snapshots/ false @@ -242,7 +242,7 @@ apache.releases - https://repository.apache.org/content/groups/public/ + http://repository.apache.org/content/groups/public/ true @@ -252,7 +252,7 @@ apache.snapshots - https://repository.apache.org/content/groups/snapshots/ + http://repository.apache.org/content/groups/snapshots/ false diff --git a/src/main/java/com/commafeed/backend/feeds/FeedFetcher.java b/src/main/java/com/commafeed/backend/feeds/FeedFetcher.java index 5605fded..fc99378c 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedFetcher.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedFetcher.java @@ -1,14 +1,6 @@ package com.commafeed.backend.feeds; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -import javax.ejb.AccessTimeout; -import javax.ejb.AsyncResult; -import javax.ejb.Asynchronous; -import javax.ejb.Lock; -import javax.ejb.LockType; -import javax.ejb.Singleton; +import javax.ejb.Stateless; import javax.inject.Inject; import org.apache.http.HttpEntity; @@ -16,14 +8,16 @@ import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.params.HttpConnectionParams; import org.apache.http.params.HttpProtocolParams; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.commafeed.backend.model.Feed; +import com.sun.syndication.io.FeedException; -@Singleton +@Stateless public class FeedFetcher { private static Logger log = LoggerFactory.getLogger(FeedFetcher.class); @@ -31,15 +25,15 @@ public class FeedFetcher { @Inject FeedParser parser; - @Asynchronous - @Lock(LockType.READ) - @AccessTimeout(value = 15, unit = TimeUnit.SECONDS) - public Future fetch(String feedUrl) { + public Feed fetch(String feedUrl) throws FeedException { log.debug("Fetching feed {}", feedUrl); Feed feed = null; HttpClient httpclient = new DefaultHttpClient(); HttpProtocolParams.setContentCharset(httpclient.getParams(), "UTF-8"); + HttpConnectionParams + .setConnectionTimeout(httpclient.getParams(), 15000); + HttpConnectionParams.setSoTimeout(httpclient.getParams(), 15000); try { HttpGet httpget = new HttpGet(feedUrl); @@ -48,11 +42,11 @@ public class FeedFetcher { String content = EntityUtils.toString(entity, "UTF-8"); feed = parser.parse(feedUrl, content); } catch (Exception e) { - throw new RuntimeException(e.getMessage(), e); + throw new FeedException(e.getMessage(), e); } finally { httpclient.getConnectionManager().shutdown(); } - return new AsyncResult(feed); + return feed; } } diff --git a/src/main/java/com/commafeed/backend/feeds/FeedParser.java b/src/main/java/com/commafeed/backend/feeds/FeedParser.java index 1b70680f..db39bdf3 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedParser.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedParser.java @@ -30,7 +30,6 @@ public class FeedParser { feed.setLastUpdated(Calendar.getInstance().getTime()); try { - xml = balanceTags(xml); SyndFeed rss = new SyndFeedInput().build(new StringReader(xml)); List items = rss.getEntries(); @@ -46,7 +45,6 @@ public class FeedParser { feed.getEntries().add(entry); } } catch (Exception e) { - e.printStackTrace(); throw new FeedException(String.format( "Could not parse feed from %s : %s", feedUrl, e.getMessage()), e); @@ -72,11 +70,6 @@ public class FeedParser { return content; } - private String balanceTags(String xml) throws Exception { - // TODO close tags - return xml; - } - private String handleContent(String content) { org.jsoup.nodes.Document doc = Jsoup.parse(content, "UTF-8"); doc.select("a").attr("target", "_blank"); diff --git a/src/main/java/com/commafeed/backend/feeds/FeedTimer.java b/src/main/java/com/commafeed/backend/feeds/FeedTimer.java index d5e55c22..00397879 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedTimer.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedTimer.java @@ -1,65 +1,25 @@ package com.commafeed.backend.feeds; -import java.util.Calendar; -import java.util.Map; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - import javax.ejb.Schedule; import javax.ejb.Singleton; import javax.inject.Inject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.commafeed.backend.dao.FeedEntryService; import com.commafeed.backend.dao.FeedService; import com.commafeed.backend.model.Feed; -import com.google.common.collect.Maps; @Singleton public class FeedTimer { - private static Logger log = LoggerFactory.getLogger(FeedTimer.class); - @Inject FeedService feedService; @Inject - FeedEntryService feedEntryService; - - @Inject - FeedFetcher fetcher; + FeedUpdater updater; @Schedule(hour = "*", minute = "*", persistent = false) private void timeout() { - Map feeds = Maps.newHashMap(); for (Feed feed : feedService.findAll()) { - feeds.put(feed.getUrl(), feed); - } - - Map> futures = Maps.newHashMap(); - for (Feed feed : feeds.values()) { - Future future = fetcher.fetch(feed.getUrl()); - futures.put(feed.getUrl(), future); - } - - for (String key : futures.keySet()) { - Future future = futures.get(key); - try { - Feed feed = future.get(1, TimeUnit.MINUTES); - feedEntryService - .updateEntries(feed.getUrl(), feed.getEntries()); - } catch (Exception e) { - log.info("Unable to refresh feed " + key + " : " - + e.getMessage()); - - Feed feed = feeds.get(key); - feed.setLastUpdated(Calendar.getInstance().getTime()); - feed.setMessage("Unable to refresh feed: " + e.getMessage()); - feedService.update(feed); - } + updater.update(feed); } } - } diff --git a/src/main/java/com/commafeed/backend/feeds/FeedUpdater.java b/src/main/java/com/commafeed/backend/feeds/FeedUpdater.java new file mode 100644 index 00000000..96f9ece4 --- /dev/null +++ b/src/main/java/com/commafeed/backend/feeds/FeedUpdater.java @@ -0,0 +1,51 @@ +package com.commafeed.backend.feeds; + +import java.util.Calendar; +import java.util.concurrent.TimeUnit; + +import javax.ejb.AccessTimeout; +import javax.ejb.Asynchronous; +import javax.ejb.Lock; +import javax.ejb.LockType; +import javax.ejb.Stateless; +import javax.inject.Inject; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.commafeed.backend.dao.FeedEntryService; +import com.commafeed.backend.dao.FeedService; +import com.commafeed.backend.model.Feed; + +@Stateless +public class FeedUpdater { + + private static Logger log = LoggerFactory.getLogger(FeedTimer.class); + + @Inject + FeedFetcher fetcher; + + @Inject + FeedService feedService; + + @Inject + FeedEntryService feedEntryService; + + @Asynchronous + @Lock(LockType.READ) + @AccessTimeout(value = 15, unit = TimeUnit.SECONDS) + public void update(Feed feed) { + + try { + Feed fetchedFeed = fetcher.fetch(feed.getUrl()); + feedEntryService.updateEntries(feed.getUrl(), + fetchedFeed.getEntries()); + } catch (Exception e) { + log.info("Unable to refresh feed " + feed.getUrl() + " : " + + e.getMessage()); + feed.setLastUpdated(Calendar.getInstance().getTime()); + feed.setMessage("Unable to refresh feed: " + e.getMessage()); + feedService.update(feed); + } + } +} diff --git a/src/main/java/com/commafeed/backend/feeds/OPMLImporter.java b/src/main/java/com/commafeed/backend/feeds/OPMLImporter.java index 2489f332..0adab124 100644 --- a/src/main/java/com/commafeed/backend/feeds/OPMLImporter.java +++ b/src/main/java/com/commafeed/backend/feeds/OPMLImporter.java @@ -3,7 +3,7 @@ package com.commafeed.backend.feeds; import java.io.StringReader; import java.util.List; -import javax.ejb.Singleton; +import javax.ejb.Stateless; import javax.inject.Inject; import org.apache.commons.lang.StringUtils; @@ -20,7 +20,7 @@ import com.sun.syndication.feed.opml.Outline; import com.sun.syndication.io.FeedException; import com.sun.syndication.io.WireFeedInput; -@Singleton +@Stateless public class OPMLImporter { @Inject