tweak feed fetching

This commit is contained in:
Jeremie Panzer
2013-03-27 09:37:15 +01:00
parent 3bf414c82b
commit 0d83bd19d0
6 changed files with 69 additions and 71 deletions

View File

@@ -1,14 +1,6 @@
package com.commafeed.backend.feeds;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import javax.ejb.AccessTimeout;
import javax.ejb.AsyncResult;
import javax.ejb.Asynchronous;
import javax.ejb.Lock;
import javax.ejb.LockType;
import javax.ejb.Singleton;
import javax.ejb.Stateless;
import javax.inject.Inject;
import org.apache.http.HttpEntity;
@@ -16,14 +8,16 @@ import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.commafeed.backend.model.Feed;
import com.sun.syndication.io.FeedException;
@Singleton
@Stateless
public class FeedFetcher {
private static Logger log = LoggerFactory.getLogger(FeedFetcher.class);
@@ -31,15 +25,15 @@ public class FeedFetcher {
@Inject
FeedParser parser;
@Asynchronous
@Lock(LockType.READ)
@AccessTimeout(value = 15, unit = TimeUnit.SECONDS)
public Future<Feed> fetch(String feedUrl) {
public Feed fetch(String feedUrl) throws FeedException {
log.debug("Fetching feed {}", feedUrl);
Feed feed = null;
HttpClient httpclient = new DefaultHttpClient();
HttpProtocolParams.setContentCharset(httpclient.getParams(), "UTF-8");
HttpConnectionParams
.setConnectionTimeout(httpclient.getParams(), 15000);
HttpConnectionParams.setSoTimeout(httpclient.getParams(), 15000);
try {
HttpGet httpget = new HttpGet(feedUrl);
@@ -48,11 +42,11 @@ public class FeedFetcher {
String content = EntityUtils.toString(entity, "UTF-8");
feed = parser.parse(feedUrl, content);
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
throw new FeedException(e.getMessage(), e);
} finally {
httpclient.getConnectionManager().shutdown();
}
return new AsyncResult<Feed>(feed);
return feed;
}
}

View File

@@ -30,7 +30,6 @@ public class FeedParser {
feed.setLastUpdated(Calendar.getInstance().getTime());
try {
xml = balanceTags(xml);
SyndFeed rss = new SyndFeedInput().build(new StringReader(xml));
List<SyndEntry> items = rss.getEntries();
@@ -46,7 +45,6 @@ public class FeedParser {
feed.getEntries().add(entry);
}
} catch (Exception e) {
e.printStackTrace();
throw new FeedException(String.format(
"Could not parse feed from %s : %s", feedUrl,
e.getMessage()), e);
@@ -72,11 +70,6 @@ public class FeedParser {
return content;
}
private String balanceTags(String xml) throws Exception {
// TODO close tags
return xml;
}
private String handleContent(String content) {
org.jsoup.nodes.Document doc = Jsoup.parse(content, "UTF-8");
doc.select("a").attr("target", "_blank");

View File

@@ -1,65 +1,25 @@
package com.commafeed.backend.feeds;
import java.util.Calendar;
import java.util.Map;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import javax.ejb.Schedule;
import javax.ejb.Singleton;
import javax.inject.Inject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.commafeed.backend.dao.FeedEntryService;
import com.commafeed.backend.dao.FeedService;
import com.commafeed.backend.model.Feed;
import com.google.common.collect.Maps;
@Singleton
public class FeedTimer {
private static Logger log = LoggerFactory.getLogger(FeedTimer.class);
@Inject
FeedService feedService;
@Inject
FeedEntryService feedEntryService;
@Inject
FeedFetcher fetcher;
FeedUpdater updater;
@Schedule(hour = "*", minute = "*", persistent = false)
private void timeout() {
Map<String, Feed> feeds = Maps.newHashMap();
for (Feed feed : feedService.findAll()) {
feeds.put(feed.getUrl(), feed);
}
Map<String, Future<Feed>> futures = Maps.newHashMap();
for (Feed feed : feeds.values()) {
Future<Feed> future = fetcher.fetch(feed.getUrl());
futures.put(feed.getUrl(), future);
}
for (String key : futures.keySet()) {
Future<Feed> future = futures.get(key);
try {
Feed feed = future.get(1, TimeUnit.MINUTES);
feedEntryService
.updateEntries(feed.getUrl(), feed.getEntries());
} catch (Exception e) {
log.info("Unable to refresh feed " + key + " : "
+ e.getMessage());
Feed feed = feeds.get(key);
feed.setLastUpdated(Calendar.getInstance().getTime());
feed.setMessage("Unable to refresh feed: " + e.getMessage());
feedService.update(feed);
}
updater.update(feed);
}
}
}

View File

@@ -0,0 +1,51 @@
package com.commafeed.backend.feeds;
import java.util.Calendar;
import java.util.concurrent.TimeUnit;
import javax.ejb.AccessTimeout;
import javax.ejb.Asynchronous;
import javax.ejb.Lock;
import javax.ejb.LockType;
import javax.ejb.Stateless;
import javax.inject.Inject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.commafeed.backend.dao.FeedEntryService;
import com.commafeed.backend.dao.FeedService;
import com.commafeed.backend.model.Feed;
@Stateless
public class FeedUpdater {
private static Logger log = LoggerFactory.getLogger(FeedTimer.class);
@Inject
FeedFetcher fetcher;
@Inject
FeedService feedService;
@Inject
FeedEntryService feedEntryService;
@Asynchronous
@Lock(LockType.READ)
@AccessTimeout(value = 15, unit = TimeUnit.SECONDS)
public void update(Feed feed) {
try {
Feed fetchedFeed = fetcher.fetch(feed.getUrl());
feedEntryService.updateEntries(feed.getUrl(),
fetchedFeed.getEntries());
} catch (Exception e) {
log.info("Unable to refresh feed " + feed.getUrl() + " : "
+ e.getMessage());
feed.setLastUpdated(Calendar.getInstance().getTime());
feed.setMessage("Unable to refresh feed: " + e.getMessage());
feedService.update(feed);
}
}
}

View File

@@ -3,7 +3,7 @@ package com.commafeed.backend.feeds;
import java.io.StringReader;
import java.util.List;
import javax.ejb.Singleton;
import javax.ejb.Stateless;
import javax.inject.Inject;
import org.apache.commons.lang.StringUtils;
@@ -20,7 +20,7 @@ import com.sun.syndication.feed.opml.Outline;
import com.sun.syndication.io.FeedException;
import com.sun.syndication.io.WireFeedInput;
@Singleton
@Stateless
public class OPMLImporter {
@Inject