mirror of
https://github.com/Athou/commafeed.git
synced 2026-03-21 21:37:29 +00:00
tweak feed fetching
This commit is contained in:
8
pom.xml
8
pom.xml
@@ -210,7 +210,7 @@
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>apache.releases</id>
|
||||
<url>https://repository.apache.org/content/groups/public/</url>
|
||||
<url>http://repository.apache.org/content/groups/public/</url>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
@@ -220,7 +220,7 @@
|
||||
</repository>
|
||||
<repository>
|
||||
<id>apache.snapshots</id>
|
||||
<url>https://repository.apache.org/content/groups/snapshots/</url>
|
||||
<url>http://repository.apache.org/content/groups/snapshots/</url>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
@@ -242,7 +242,7 @@
|
||||
<pluginRepositories>
|
||||
<pluginRepository>
|
||||
<id>apache.releases</id>
|
||||
<url>https://repository.apache.org/content/groups/public/</url>
|
||||
<url>http://repository.apache.org/content/groups/public/</url>
|
||||
<releases>
|
||||
<enabled>true</enabled>
|
||||
</releases>
|
||||
@@ -252,7 +252,7 @@
|
||||
</pluginRepository>
|
||||
<pluginRepository>
|
||||
<id>apache.snapshots</id>
|
||||
<url>https://repository.apache.org/content/groups/snapshots/</url>
|
||||
<url>http://repository.apache.org/content/groups/snapshots/</url>
|
||||
<releases>
|
||||
<enabled>false</enabled>
|
||||
</releases>
|
||||
|
||||
@@ -1,14 +1,6 @@
|
||||
package com.commafeed.backend.feeds;
|
||||
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.ejb.AccessTimeout;
|
||||
import javax.ejb.AsyncResult;
|
||||
import javax.ejb.Asynchronous;
|
||||
import javax.ejb.Lock;
|
||||
import javax.ejb.LockType;
|
||||
import javax.ejb.Singleton;
|
||||
import javax.ejb.Stateless;
|
||||
import javax.inject.Inject;
|
||||
|
||||
import org.apache.http.HttpEntity;
|
||||
@@ -16,14 +8,16 @@ import org.apache.http.HttpResponse;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.apache.http.params.HttpConnectionParams;
|
||||
import org.apache.http.params.HttpProtocolParams;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.commafeed.backend.model.Feed;
|
||||
import com.sun.syndication.io.FeedException;
|
||||
|
||||
@Singleton
|
||||
@Stateless
|
||||
public class FeedFetcher {
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(FeedFetcher.class);
|
||||
@@ -31,15 +25,15 @@ public class FeedFetcher {
|
||||
@Inject
|
||||
FeedParser parser;
|
||||
|
||||
@Asynchronous
|
||||
@Lock(LockType.READ)
|
||||
@AccessTimeout(value = 15, unit = TimeUnit.SECONDS)
|
||||
public Future<Feed> fetch(String feedUrl) {
|
||||
public Feed fetch(String feedUrl) throws FeedException {
|
||||
log.debug("Fetching feed {}", feedUrl);
|
||||
Feed feed = null;
|
||||
|
||||
HttpClient httpclient = new DefaultHttpClient();
|
||||
HttpProtocolParams.setContentCharset(httpclient.getParams(), "UTF-8");
|
||||
HttpConnectionParams
|
||||
.setConnectionTimeout(httpclient.getParams(), 15000);
|
||||
HttpConnectionParams.setSoTimeout(httpclient.getParams(), 15000);
|
||||
|
||||
try {
|
||||
HttpGet httpget = new HttpGet(feedUrl);
|
||||
@@ -48,11 +42,11 @@ public class FeedFetcher {
|
||||
String content = EntityUtils.toString(entity, "UTF-8");
|
||||
feed = parser.parse(feedUrl, content);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e.getMessage(), e);
|
||||
throw new FeedException(e.getMessage(), e);
|
||||
} finally {
|
||||
httpclient.getConnectionManager().shutdown();
|
||||
}
|
||||
return new AsyncResult<Feed>(feed);
|
||||
return feed;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -30,7 +30,6 @@ public class FeedParser {
|
||||
feed.setLastUpdated(Calendar.getInstance().getTime());
|
||||
|
||||
try {
|
||||
xml = balanceTags(xml);
|
||||
SyndFeed rss = new SyndFeedInput().build(new StringReader(xml));
|
||||
|
||||
List<SyndEntry> items = rss.getEntries();
|
||||
@@ -46,7 +45,6 @@ public class FeedParser {
|
||||
feed.getEntries().add(entry);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
throw new FeedException(String.format(
|
||||
"Could not parse feed from %s : %s", feedUrl,
|
||||
e.getMessage()), e);
|
||||
@@ -72,11 +70,6 @@ public class FeedParser {
|
||||
return content;
|
||||
}
|
||||
|
||||
private String balanceTags(String xml) throws Exception {
|
||||
// TODO close tags
|
||||
return xml;
|
||||
}
|
||||
|
||||
private String handleContent(String content) {
|
||||
org.jsoup.nodes.Document doc = Jsoup.parse(content, "UTF-8");
|
||||
doc.select("a").attr("target", "_blank");
|
||||
|
||||
@@ -1,65 +1,25 @@
|
||||
package com.commafeed.backend.feeds;
|
||||
|
||||
import java.util.Calendar;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.ejb.Schedule;
|
||||
import javax.ejb.Singleton;
|
||||
import javax.inject.Inject;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.commafeed.backend.dao.FeedEntryService;
|
||||
import com.commafeed.backend.dao.FeedService;
|
||||
import com.commafeed.backend.model.Feed;
|
||||
import com.google.common.collect.Maps;
|
||||
|
||||
@Singleton
|
||||
public class FeedTimer {
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(FeedTimer.class);
|
||||
|
||||
@Inject
|
||||
FeedService feedService;
|
||||
|
||||
@Inject
|
||||
FeedEntryService feedEntryService;
|
||||
|
||||
@Inject
|
||||
FeedFetcher fetcher;
|
||||
FeedUpdater updater;
|
||||
|
||||
@Schedule(hour = "*", minute = "*", persistent = false)
|
||||
private void timeout() {
|
||||
Map<String, Feed> feeds = Maps.newHashMap();
|
||||
for (Feed feed : feedService.findAll()) {
|
||||
feeds.put(feed.getUrl(), feed);
|
||||
}
|
||||
|
||||
Map<String, Future<Feed>> futures = Maps.newHashMap();
|
||||
for (Feed feed : feeds.values()) {
|
||||
Future<Feed> future = fetcher.fetch(feed.getUrl());
|
||||
futures.put(feed.getUrl(), future);
|
||||
}
|
||||
|
||||
for (String key : futures.keySet()) {
|
||||
Future<Feed> future = futures.get(key);
|
||||
try {
|
||||
Feed feed = future.get(1, TimeUnit.MINUTES);
|
||||
feedEntryService
|
||||
.updateEntries(feed.getUrl(), feed.getEntries());
|
||||
} catch (Exception e) {
|
||||
log.info("Unable to refresh feed " + key + " : "
|
||||
+ e.getMessage());
|
||||
|
||||
Feed feed = feeds.get(key);
|
||||
feed.setLastUpdated(Calendar.getInstance().getTime());
|
||||
feed.setMessage("Unable to refresh feed: " + e.getMessage());
|
||||
feedService.update(feed);
|
||||
}
|
||||
updater.update(feed);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
51
src/main/java/com/commafeed/backend/feeds/FeedUpdater.java
Normal file
51
src/main/java/com/commafeed/backend/feeds/FeedUpdater.java
Normal file
@@ -0,0 +1,51 @@
|
||||
package com.commafeed.backend.feeds;
|
||||
|
||||
import java.util.Calendar;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.ejb.AccessTimeout;
|
||||
import javax.ejb.Asynchronous;
|
||||
import javax.ejb.Lock;
|
||||
import javax.ejb.LockType;
|
||||
import javax.ejb.Stateless;
|
||||
import javax.inject.Inject;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.commafeed.backend.dao.FeedEntryService;
|
||||
import com.commafeed.backend.dao.FeedService;
|
||||
import com.commafeed.backend.model.Feed;
|
||||
|
||||
@Stateless
|
||||
public class FeedUpdater {
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(FeedTimer.class);
|
||||
|
||||
@Inject
|
||||
FeedFetcher fetcher;
|
||||
|
||||
@Inject
|
||||
FeedService feedService;
|
||||
|
||||
@Inject
|
||||
FeedEntryService feedEntryService;
|
||||
|
||||
@Asynchronous
|
||||
@Lock(LockType.READ)
|
||||
@AccessTimeout(value = 15, unit = TimeUnit.SECONDS)
|
||||
public void update(Feed feed) {
|
||||
|
||||
try {
|
||||
Feed fetchedFeed = fetcher.fetch(feed.getUrl());
|
||||
feedEntryService.updateEntries(feed.getUrl(),
|
||||
fetchedFeed.getEntries());
|
||||
} catch (Exception e) {
|
||||
log.info("Unable to refresh feed " + feed.getUrl() + " : "
|
||||
+ e.getMessage());
|
||||
feed.setLastUpdated(Calendar.getInstance().getTime());
|
||||
feed.setMessage("Unable to refresh feed: " + e.getMessage());
|
||||
feedService.update(feed);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ package com.commafeed.backend.feeds;
|
||||
import java.io.StringReader;
|
||||
import java.util.List;
|
||||
|
||||
import javax.ejb.Singleton;
|
||||
import javax.ejb.Stateless;
|
||||
import javax.inject.Inject;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
@@ -20,7 +20,7 @@ import com.sun.syndication.feed.opml.Outline;
|
||||
import com.sun.syndication.io.FeedException;
|
||||
import com.sun.syndication.io.WireFeedInput;
|
||||
|
||||
@Singleton
|
||||
@Stateless
|
||||
public class OPMLImporter {
|
||||
|
||||
@Inject
|
||||
|
||||
Reference in New Issue
Block a user