forked from Archives/Athou_commafeed
tweak feed fetching
This commit is contained in:
8
pom.xml
8
pom.xml
@@ -210,7 +210,7 @@
|
|||||||
<repositories>
|
<repositories>
|
||||||
<repository>
|
<repository>
|
||||||
<id>apache.releases</id>
|
<id>apache.releases</id>
|
||||||
<url>https://repository.apache.org/content/groups/public/</url>
|
<url>http://repository.apache.org/content/groups/public/</url>
|
||||||
<releases>
|
<releases>
|
||||||
<enabled>true</enabled>
|
<enabled>true</enabled>
|
||||||
</releases>
|
</releases>
|
||||||
@@ -220,7 +220,7 @@
|
|||||||
</repository>
|
</repository>
|
||||||
<repository>
|
<repository>
|
||||||
<id>apache.snapshots</id>
|
<id>apache.snapshots</id>
|
||||||
<url>https://repository.apache.org/content/groups/snapshots/</url>
|
<url>http://repository.apache.org/content/groups/snapshots/</url>
|
||||||
<releases>
|
<releases>
|
||||||
<enabled>false</enabled>
|
<enabled>false</enabled>
|
||||||
</releases>
|
</releases>
|
||||||
@@ -242,7 +242,7 @@
|
|||||||
<pluginRepositories>
|
<pluginRepositories>
|
||||||
<pluginRepository>
|
<pluginRepository>
|
||||||
<id>apache.releases</id>
|
<id>apache.releases</id>
|
||||||
<url>https://repository.apache.org/content/groups/public/</url>
|
<url>http://repository.apache.org/content/groups/public/</url>
|
||||||
<releases>
|
<releases>
|
||||||
<enabled>true</enabled>
|
<enabled>true</enabled>
|
||||||
</releases>
|
</releases>
|
||||||
@@ -252,7 +252,7 @@
|
|||||||
</pluginRepository>
|
</pluginRepository>
|
||||||
<pluginRepository>
|
<pluginRepository>
|
||||||
<id>apache.snapshots</id>
|
<id>apache.snapshots</id>
|
||||||
<url>https://repository.apache.org/content/groups/snapshots/</url>
|
<url>http://repository.apache.org/content/groups/snapshots/</url>
|
||||||
<releases>
|
<releases>
|
||||||
<enabled>false</enabled>
|
<enabled>false</enabled>
|
||||||
</releases>
|
</releases>
|
||||||
|
|||||||
@@ -1,14 +1,6 @@
|
|||||||
package com.commafeed.backend.feeds;
|
package com.commafeed.backend.feeds;
|
||||||
|
|
||||||
import java.util.concurrent.Future;
|
import javax.ejb.Stateless;
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
import javax.ejb.AccessTimeout;
|
|
||||||
import javax.ejb.AsyncResult;
|
|
||||||
import javax.ejb.Asynchronous;
|
|
||||||
import javax.ejb.Lock;
|
|
||||||
import javax.ejb.LockType;
|
|
||||||
import javax.ejb.Singleton;
|
|
||||||
import javax.inject.Inject;
|
import javax.inject.Inject;
|
||||||
|
|
||||||
import org.apache.http.HttpEntity;
|
import org.apache.http.HttpEntity;
|
||||||
@@ -16,14 +8,16 @@ import org.apache.http.HttpResponse;
|
|||||||
import org.apache.http.client.HttpClient;
|
import org.apache.http.client.HttpClient;
|
||||||
import org.apache.http.client.methods.HttpGet;
|
import org.apache.http.client.methods.HttpGet;
|
||||||
import org.apache.http.impl.client.DefaultHttpClient;
|
import org.apache.http.impl.client.DefaultHttpClient;
|
||||||
|
import org.apache.http.params.HttpConnectionParams;
|
||||||
import org.apache.http.params.HttpProtocolParams;
|
import org.apache.http.params.HttpProtocolParams;
|
||||||
import org.apache.http.util.EntityUtils;
|
import org.apache.http.util.EntityUtils;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.commafeed.backend.model.Feed;
|
import com.commafeed.backend.model.Feed;
|
||||||
|
import com.sun.syndication.io.FeedException;
|
||||||
|
|
||||||
@Singleton
|
@Stateless
|
||||||
public class FeedFetcher {
|
public class FeedFetcher {
|
||||||
|
|
||||||
private static Logger log = LoggerFactory.getLogger(FeedFetcher.class);
|
private static Logger log = LoggerFactory.getLogger(FeedFetcher.class);
|
||||||
@@ -31,15 +25,15 @@ public class FeedFetcher {
|
|||||||
@Inject
|
@Inject
|
||||||
FeedParser parser;
|
FeedParser parser;
|
||||||
|
|
||||||
@Asynchronous
|
public Feed fetch(String feedUrl) throws FeedException {
|
||||||
@Lock(LockType.READ)
|
|
||||||
@AccessTimeout(value = 15, unit = TimeUnit.SECONDS)
|
|
||||||
public Future<Feed> fetch(String feedUrl) {
|
|
||||||
log.debug("Fetching feed {}", feedUrl);
|
log.debug("Fetching feed {}", feedUrl);
|
||||||
Feed feed = null;
|
Feed feed = null;
|
||||||
|
|
||||||
HttpClient httpclient = new DefaultHttpClient();
|
HttpClient httpclient = new DefaultHttpClient();
|
||||||
HttpProtocolParams.setContentCharset(httpclient.getParams(), "UTF-8");
|
HttpProtocolParams.setContentCharset(httpclient.getParams(), "UTF-8");
|
||||||
|
HttpConnectionParams
|
||||||
|
.setConnectionTimeout(httpclient.getParams(), 15000);
|
||||||
|
HttpConnectionParams.setSoTimeout(httpclient.getParams(), 15000);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
HttpGet httpget = new HttpGet(feedUrl);
|
HttpGet httpget = new HttpGet(feedUrl);
|
||||||
@@ -48,11 +42,11 @@ public class FeedFetcher {
|
|||||||
String content = EntityUtils.toString(entity, "UTF-8");
|
String content = EntityUtils.toString(entity, "UTF-8");
|
||||||
feed = parser.parse(feedUrl, content);
|
feed = parser.parse(feedUrl, content);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new RuntimeException(e.getMessage(), e);
|
throw new FeedException(e.getMessage(), e);
|
||||||
} finally {
|
} finally {
|
||||||
httpclient.getConnectionManager().shutdown();
|
httpclient.getConnectionManager().shutdown();
|
||||||
}
|
}
|
||||||
return new AsyncResult<Feed>(feed);
|
return feed;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,7 +30,6 @@ public class FeedParser {
|
|||||||
feed.setLastUpdated(Calendar.getInstance().getTime());
|
feed.setLastUpdated(Calendar.getInstance().getTime());
|
||||||
|
|
||||||
try {
|
try {
|
||||||
xml = balanceTags(xml);
|
|
||||||
SyndFeed rss = new SyndFeedInput().build(new StringReader(xml));
|
SyndFeed rss = new SyndFeedInput().build(new StringReader(xml));
|
||||||
|
|
||||||
List<SyndEntry> items = rss.getEntries();
|
List<SyndEntry> items = rss.getEntries();
|
||||||
@@ -46,7 +45,6 @@ public class FeedParser {
|
|||||||
feed.getEntries().add(entry);
|
feed.getEntries().add(entry);
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
|
||||||
throw new FeedException(String.format(
|
throw new FeedException(String.format(
|
||||||
"Could not parse feed from %s : %s", feedUrl,
|
"Could not parse feed from %s : %s", feedUrl,
|
||||||
e.getMessage()), e);
|
e.getMessage()), e);
|
||||||
@@ -72,11 +70,6 @@ public class FeedParser {
|
|||||||
return content;
|
return content;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String balanceTags(String xml) throws Exception {
|
|
||||||
// TODO close tags
|
|
||||||
return xml;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String handleContent(String content) {
|
private String handleContent(String content) {
|
||||||
org.jsoup.nodes.Document doc = Jsoup.parse(content, "UTF-8");
|
org.jsoup.nodes.Document doc = Jsoup.parse(content, "UTF-8");
|
||||||
doc.select("a").attr("target", "_blank");
|
doc.select("a").attr("target", "_blank");
|
||||||
|
|||||||
@@ -1,65 +1,25 @@
|
|||||||
package com.commafeed.backend.feeds;
|
package com.commafeed.backend.feeds;
|
||||||
|
|
||||||
import java.util.Calendar;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.Future;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
import javax.ejb.Schedule;
|
import javax.ejb.Schedule;
|
||||||
import javax.ejb.Singleton;
|
import javax.ejb.Singleton;
|
||||||
import javax.inject.Inject;
|
import javax.inject.Inject;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import com.commafeed.backend.dao.FeedEntryService;
|
|
||||||
import com.commafeed.backend.dao.FeedService;
|
import com.commafeed.backend.dao.FeedService;
|
||||||
import com.commafeed.backend.model.Feed;
|
import com.commafeed.backend.model.Feed;
|
||||||
import com.google.common.collect.Maps;
|
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
public class FeedTimer {
|
public class FeedTimer {
|
||||||
|
|
||||||
private static Logger log = LoggerFactory.getLogger(FeedTimer.class);
|
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
FeedService feedService;
|
FeedService feedService;
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
FeedEntryService feedEntryService;
|
FeedUpdater updater;
|
||||||
|
|
||||||
@Inject
|
|
||||||
FeedFetcher fetcher;
|
|
||||||
|
|
||||||
@Schedule(hour = "*", minute = "*", persistent = false)
|
@Schedule(hour = "*", minute = "*", persistent = false)
|
||||||
private void timeout() {
|
private void timeout() {
|
||||||
Map<String, Feed> feeds = Maps.newHashMap();
|
|
||||||
for (Feed feed : feedService.findAll()) {
|
for (Feed feed : feedService.findAll()) {
|
||||||
feeds.put(feed.getUrl(), feed);
|
updater.update(feed);
|
||||||
}
|
|
||||||
|
|
||||||
Map<String, Future<Feed>> futures = Maps.newHashMap();
|
|
||||||
for (Feed feed : feeds.values()) {
|
|
||||||
Future<Feed> future = fetcher.fetch(feed.getUrl());
|
|
||||||
futures.put(feed.getUrl(), future);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (String key : futures.keySet()) {
|
|
||||||
Future<Feed> future = futures.get(key);
|
|
||||||
try {
|
|
||||||
Feed feed = future.get(1, TimeUnit.MINUTES);
|
|
||||||
feedEntryService
|
|
||||||
.updateEntries(feed.getUrl(), feed.getEntries());
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.info("Unable to refresh feed " + key + " : "
|
|
||||||
+ e.getMessage());
|
|
||||||
|
|
||||||
Feed feed = feeds.get(key);
|
|
||||||
feed.setLastUpdated(Calendar.getInstance().getTime());
|
|
||||||
feed.setMessage("Unable to refresh feed: " + e.getMessage());
|
|
||||||
feedService.update(feed);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
51
src/main/java/com/commafeed/backend/feeds/FeedUpdater.java
Normal file
51
src/main/java/com/commafeed/backend/feeds/FeedUpdater.java
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
package com.commafeed.backend.feeds;
|
||||||
|
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import javax.ejb.AccessTimeout;
|
||||||
|
import javax.ejb.Asynchronous;
|
||||||
|
import javax.ejb.Lock;
|
||||||
|
import javax.ejb.LockType;
|
||||||
|
import javax.ejb.Stateless;
|
||||||
|
import javax.inject.Inject;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.commafeed.backend.dao.FeedEntryService;
|
||||||
|
import com.commafeed.backend.dao.FeedService;
|
||||||
|
import com.commafeed.backend.model.Feed;
|
||||||
|
|
||||||
|
@Stateless
|
||||||
|
public class FeedUpdater {
|
||||||
|
|
||||||
|
private static Logger log = LoggerFactory.getLogger(FeedTimer.class);
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
FeedFetcher fetcher;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
FeedService feedService;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
FeedEntryService feedEntryService;
|
||||||
|
|
||||||
|
@Asynchronous
|
||||||
|
@Lock(LockType.READ)
|
||||||
|
@AccessTimeout(value = 15, unit = TimeUnit.SECONDS)
|
||||||
|
public void update(Feed feed) {
|
||||||
|
|
||||||
|
try {
|
||||||
|
Feed fetchedFeed = fetcher.fetch(feed.getUrl());
|
||||||
|
feedEntryService.updateEntries(feed.getUrl(),
|
||||||
|
fetchedFeed.getEntries());
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.info("Unable to refresh feed " + feed.getUrl() + " : "
|
||||||
|
+ e.getMessage());
|
||||||
|
feed.setLastUpdated(Calendar.getInstance().getTime());
|
||||||
|
feed.setMessage("Unable to refresh feed: " + e.getMessage());
|
||||||
|
feedService.update(feed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@ package com.commafeed.backend.feeds;
|
|||||||
import java.io.StringReader;
|
import java.io.StringReader;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import javax.ejb.Singleton;
|
import javax.ejb.Stateless;
|
||||||
import javax.inject.Inject;
|
import javax.inject.Inject;
|
||||||
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
import org.apache.commons.lang.StringUtils;
|
||||||
@@ -20,7 +20,7 @@ import com.sun.syndication.feed.opml.Outline;
|
|||||||
import com.sun.syndication.io.FeedException;
|
import com.sun.syndication.io.FeedException;
|
||||||
import com.sun.syndication.io.WireFeedInput;
|
import com.sun.syndication.io.WireFeedInput;
|
||||||
|
|
||||||
@Singleton
|
@Stateless
|
||||||
public class OPMLImporter {
|
public class OPMLImporter {
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
|
|||||||
Reference in New Issue
Block a user