mirror of
https://github.com/Athou/commafeed.git
synced 2026-03-21 21:37:29 +00:00
new fetchedfeed class
This commit is contained in:
@@ -28,11 +28,11 @@ public class FeedFetcher {
|
|||||||
@Inject
|
@Inject
|
||||||
HttpGetter getter;
|
HttpGetter getter;
|
||||||
|
|
||||||
public Feed fetch(String feedUrl, boolean extractFeedUrlFromHtml,
|
public FetchedFeed fetch(String feedUrl, boolean extractFeedUrlFromHtml,
|
||||||
String lastModified, String eTag) throws FeedException,
|
String lastModified, String eTag) throws FeedException,
|
||||||
ClientProtocolException, IOException, NotModifiedException {
|
ClientProtocolException, IOException, NotModifiedException {
|
||||||
log.debug("Fetching feed {}", feedUrl);
|
log.debug("Fetching feed {}", feedUrl);
|
||||||
Feed feed = null;
|
FetchedFeed fetchedFeed = null;
|
||||||
|
|
||||||
HttpResult result = getter.getBinary(feedUrl, lastModified, eTag);
|
HttpResult result = getter.getBinary(feedUrl, lastModified, eTag);
|
||||||
if (extractFeedUrlFromHtml) {
|
if (extractFeedUrlFromHtml) {
|
||||||
@@ -43,12 +43,12 @@ public class FeedFetcher {
|
|||||||
feedUrl = extractedUrl;
|
feedUrl = extractedUrl;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
feed = parser.parse(feedUrl, result.getContent());
|
fetchedFeed = parser.parse(feedUrl, result.getContent());
|
||||||
|
Feed feed = fetchedFeed.getFeed();
|
||||||
feed.setLastModifiedHeader(result.getLastModifiedSince());
|
feed.setLastModifiedHeader(result.getLastModifiedSince());
|
||||||
feed.setEtagHeader(result.geteTag());
|
feed.setEtagHeader(result.geteTag());
|
||||||
feed.setFetchDuration(result.getDuration());
|
fetchedFeed.setFetchDuration(result.getDuration());
|
||||||
return feed;
|
return fetchedFeed;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String extractFeedUrl(String html, String baseUri) {
|
private String extractFeedUrl(String html, String baseUri) {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package com.commafeed.backend.feeds;
|
|||||||
|
|
||||||
import java.io.StringReader;
|
import java.io.StringReader;
|
||||||
import java.util.Calendar;
|
import java.util.Calendar;
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@@ -35,8 +36,10 @@ public class FeedParser {
|
|||||||
};
|
};
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public Feed parse(String feedUrl, byte[] xml) throws FeedException {
|
public FetchedFeed parse(String feedUrl, byte[] xml) throws FeedException {
|
||||||
Feed feed = new Feed();
|
FetchedFeed fetchedFeed = new FetchedFeed();
|
||||||
|
Feed feed = fetchedFeed.getFeed();
|
||||||
|
Collection<FeedEntry> entries = fetchedFeed.getEntries();
|
||||||
feed.setLastUpdated(Calendar.getInstance().getTime());
|
feed.setLastUpdated(Calendar.getInstance().getTime());
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -47,8 +50,8 @@ public class FeedParser {
|
|||||||
InputSource source = new InputSource(new StringReader(xmlString));
|
InputSource source = new InputSource(new StringReader(xmlString));
|
||||||
|
|
||||||
SyndFeed rss = new SyndFeedInput().build(source);
|
SyndFeed rss = new SyndFeedInput().build(source);
|
||||||
|
fetchedFeed.setTitle(rss.getTitle());
|
||||||
feed.setUrl(feedUrl);
|
feed.setUrl(feedUrl);
|
||||||
feed.setTitle(rss.getTitle());
|
|
||||||
feed.setLink(rss.getLink());
|
feed.setLink(rss.getLink());
|
||||||
List<SyndEntry> items = rss.getEntries();
|
List<SyndEntry> items = rss.getEntries();
|
||||||
for (SyndEntry item : items) {
|
for (SyndEntry item : items) {
|
||||||
@@ -69,21 +72,21 @@ public class FeedParser {
|
|||||||
}
|
}
|
||||||
entry.setContent(content);
|
entry.setContent(content);
|
||||||
|
|
||||||
feed.getEntries().add(entry);
|
entries.add(entry);
|
||||||
}
|
}
|
||||||
Date publishedDate = validateDate(rss.getPublishedDate());
|
Date publishedDate = validateDate(rss.getPublishedDate());
|
||||||
if (publishedDate == null && !feed.getEntries().isEmpty()) {
|
if (publishedDate == null && !feed.getEntries().isEmpty()) {
|
||||||
FeedEntry first = feed.getEntries().iterator().next();
|
FeedEntry first = entries.iterator().next();
|
||||||
publishedDate = first.getUpdated();
|
publishedDate = first.getUpdated();
|
||||||
}
|
}
|
||||||
feed.setPublishedDate(publishedDate);
|
fetchedFeed.setPublishedDate(publishedDate);
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new FeedException(String.format(
|
throw new FeedException(String.format(
|
||||||
"Could not parse feed from %s : %s", feedUrl,
|
"Could not parse feed from %s : %s", feedUrl,
|
||||||
e.getMessage()), e);
|
e.getMessage()), e);
|
||||||
}
|
}
|
||||||
return feed;
|
return fetchedFeed;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Date getUpdateDate(SyndEntry item) {
|
private Date getUpdateDate(SyndEntry item) {
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ public class FeedRefreshWorker {
|
|||||||
int errorCount = 0;
|
int errorCount = 0;
|
||||||
Date disabledUntil = null;
|
Date disabledUntil = null;
|
||||||
|
|
||||||
Feed fetchedFeed = null;
|
FetchedFeed fetchedFeed = null;
|
||||||
boolean modified = true;
|
boolean modified = true;
|
||||||
try {
|
try {
|
||||||
fetchedFeed = fetcher.fetch(feed.getUrl(), false,
|
fetchedFeed = fetcher.fetch(feed.getUrl(), false,
|
||||||
@@ -127,9 +127,10 @@ public class FeedRefreshWorker {
|
|||||||
|
|
||||||
Collection<FeedEntry> entries = null;
|
Collection<FeedEntry> entries = null;
|
||||||
if (fetchedFeed != null) {
|
if (fetchedFeed != null) {
|
||||||
feed.setLink(fetchedFeed.getLink());
|
feed.setLink(fetchedFeed.getFeed().getLink());
|
||||||
feed.setLastModifiedHeader(fetchedFeed.getLastModifiedHeader());
|
feed.setLastModifiedHeader(fetchedFeed.getFeed()
|
||||||
feed.setEtagHeader(fetchedFeed.getEtagHeader());
|
.getLastModifiedHeader());
|
||||||
|
feed.setEtagHeader(fetchedFeed.getFeed().getEtagHeader());
|
||||||
entries = fetchedFeed.getEntries();
|
entries = fetchedFeed.getEntries();
|
||||||
}
|
}
|
||||||
FeedRefreshTask task = new FeedRefreshTask(feed, entries);
|
FeedRefreshTask task = new FeedRefreshTask(feed, entries);
|
||||||
|
|||||||
59
src/main/java/com/commafeed/backend/feeds/FetchedFeed.java
Normal file
59
src/main/java/com/commafeed/backend/feeds/FetchedFeed.java
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
package com.commafeed.backend.feeds;
|
||||||
|
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
import com.commafeed.backend.model.Feed;
|
||||||
|
import com.commafeed.backend.model.FeedEntry;
|
||||||
|
import com.google.api.client.util.Lists;
|
||||||
|
|
||||||
|
public class FetchedFeed {
|
||||||
|
|
||||||
|
private Feed feed = new Feed();
|
||||||
|
private Collection<FeedEntry> entries = Lists.newArrayList();
|
||||||
|
|
||||||
|
private String title;
|
||||||
|
private long fetchDuration;
|
||||||
|
private Date publishedDate;
|
||||||
|
|
||||||
|
public Feed getFeed() {
|
||||||
|
return feed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFeed(Feed feed) {
|
||||||
|
this.feed = feed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Collection<FeedEntry> getEntries() {
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setEntries(Collection<FeedEntry> entries) {
|
||||||
|
this.entries = entries;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTitle() {
|
||||||
|
return title;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTitle(String title) {
|
||||||
|
this.title = title;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getFetchDuration() {
|
||||||
|
return fetchDuration;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFetchDuration(long fetchDuration) {
|
||||||
|
this.fetchDuration = fetchDuration;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Date getPublishedDate() {
|
||||||
|
return publishedDate;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPublishedDate(Date publishedDate) {
|
||||||
|
this.publishedDate = publishedDate;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -10,7 +10,6 @@ import javax.persistence.OneToMany;
|
|||||||
import javax.persistence.Table;
|
import javax.persistence.Table;
|
||||||
import javax.persistence.Temporal;
|
import javax.persistence.Temporal;
|
||||||
import javax.persistence.TemporalType;
|
import javax.persistence.TemporalType;
|
||||||
import javax.persistence.Transient;
|
|
||||||
|
|
||||||
import org.hibernate.annotations.Index;
|
import org.hibernate.annotations.Index;
|
||||||
|
|
||||||
@@ -31,25 +30,6 @@ public class Feed extends AbstractModel {
|
|||||||
@Index(name = "urlHash_index")
|
@Index(name = "urlHash_index")
|
||||||
private String urlHash;
|
private String urlHash;
|
||||||
|
|
||||||
/**
|
|
||||||
* title of the feed, used only when fetching, not stored
|
|
||||||
*/
|
|
||||||
@Transient
|
|
||||||
private String title;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* time it took to fetch the feed, used only when fetching, not stored
|
|
||||||
*/
|
|
||||||
@Transient
|
|
||||||
private long fetchDuration;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* extracted published date from the feed, used only when fetching, not
|
|
||||||
* stored
|
|
||||||
*/
|
|
||||||
@Transient
|
|
||||||
private Date publishedDate;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The url of the website, extracted from the feed
|
* The url of the website, extracted from the feed
|
||||||
*/
|
*/
|
||||||
@@ -138,14 +118,6 @@ public class Feed extends AbstractModel {
|
|||||||
this.subscriptions = subscriptions;
|
this.subscriptions = subscriptions;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getTitle() {
|
|
||||||
return title;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setTitle(String title) {
|
|
||||||
this.title = title;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getLink() {
|
public String getLink() {
|
||||||
return link;
|
return link;
|
||||||
}
|
}
|
||||||
@@ -194,22 +166,6 @@ public class Feed extends AbstractModel {
|
|||||||
this.etagHeader = etagHeader;
|
this.etagHeader = etagHeader;
|
||||||
}
|
}
|
||||||
|
|
||||||
public long getFetchDuration() {
|
|
||||||
return fetchDuration;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setFetchDuration(long fetchDuration) {
|
|
||||||
this.fetchDuration = fetchDuration;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Date getPublishedDate() {
|
|
||||||
return publishedDate;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setPublishedDate(Date publishedDate) {
|
|
||||||
this.publishedDate = publishedDate;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Date getLastUpdateSuccess() {
|
public Date getLastUpdateSuccess() {
|
||||||
return lastUpdateSuccess;
|
return lastUpdateSuccess;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ public class FeedREST extends AbstractResourceREST {
|
|||||||
url = prependHttp(url);
|
url = prependHttp(url);
|
||||||
Feed feed = null;
|
Feed feed = null;
|
||||||
try {
|
try {
|
||||||
feed = feedFetcher.fetch(url, true, null, null);
|
feed = feedFetcher.fetch(url, true, null, null).getFeed();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new WebApplicationException(e, Response
|
throw new WebApplicationException(e, Response
|
||||||
.status(Status.INTERNAL_SERVER_ERROR)
|
.status(Status.INTERNAL_SERVER_ERROR)
|
||||||
|
|||||||
Reference in New Issue
Block a user