Merge remote-tracking branch 'origin/master' into tg

src/main/java/com/commafeed/backend/feeds/FeedFetcher.java

@@ -1,6 +1,7 @@
 package com.commafeed.backend.feeds;
 
 import java.io.IOException;
+import java.util.Date;
 
 import javax.inject.Inject;
 
@@ -29,8 +30,9 @@ public class FeedFetcher {
 	HttpGetter getter;
 
 	public FetchedFeed fetch(String feedUrl, boolean extractFeedUrlFromHtml,
-			String lastModified, String eTag) throws FeedException,
-			ClientProtocolException, IOException, NotModifiedException {
+			String lastModified, String eTag, Date lastPublishedDate)
+			throws FeedException, ClientProtocolException, IOException,
+			NotModifiedException {
 		log.debug("Fetching feed {}", feedUrl);
 		FetchedFeed fetchedFeed = null;
 
@@ -48,6 +50,15 @@ public class FeedFetcher {
 		}
 
 		fetchedFeed = parser.parse(feedUrl, result.getContent());
+
+		if (lastPublishedDate != null
+				&& fetchedFeed.getFeed().getLastPublishedDate() != null
+				&& lastPublishedDate.getTime() == fetchedFeed.getFeed()
+						.getLastPublishedDate().getTime()) {
+			log.info("using publishedDate!");
+			throw new NotModifiedException();
+		}
+
 		Feed feed = fetchedFeed.getFeed();
 		feed.setLastModifiedHeader(result.getLastModifiedSince());
 		feed.setEtagHeader(FeedUtils.truncate(result.geteTag(), 255));

src/main/java/com/commafeed/backend/feeds/FeedParser.java

@@ -71,6 +71,7 @@ public class FeedParser {
 			fetchedFeed.setTopic(findSelf(rss));
 			feed.setUrl(feedUrl);
 			feed.setLink(rss.getLink());
+			feed.setLastPublishedDate(rss.getPublishedDate());
 			List<SyndEntry> items = rss.getEntries();
 			for (SyndEntry item : items) {
 				FeedEntry entry = new FeedEntry();
@@ -105,12 +106,12 @@ public class FeedParser {
 
 				entries.add(entry);
 			}
-			Date publishedDate = null;
+			Date lastEntryDate = null;
 			if (!entries.isEmpty()) {
 				Long timestamp = FeedUtils.getSortedTimestamps(entries).get(0);
-				publishedDate = new Date(timestamp);
+				lastEntryDate = new Date(timestamp);
 			}
-			fetchedFeed.setPublishedDate(publishedDate);
+			fetchedFeed.setLastEntryDate(lastEntryDate);
 
 		} catch (Exception e) {
 			throw new FeedException(String.format(

src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java

@@ -82,7 +82,8 @@ public class FeedRefreshWorker {
 		Date now = Calendar.getInstance().getTime();
 		try {
 			FetchedFeed fetchedFeed = fetcher.fetch(feed.getUrl(), false,
-					feed.getLastModifiedHeader(), feed.getEtagHeader());
+					feed.getLastModifiedHeader(), feed.getEtagHeader(),
+					feed.getLastPublishedDate());
 			// stops here if NotModifiedException or any other exception is
 			// thrown
 			List<FeedEntry> entries = fetchedFeed.getEntries();
@@ -90,7 +91,7 @@ public class FeedRefreshWorker {
 			Date disabledUntil = null;
 			if (applicationSettingsService.get().isHeavyLoad()) {
 				disabledUntil = FeedUtils.buildDisabledUntil(
-						fetchedFeed.getPublishedDate(), entries);
+						fetchedFeed.getLastEntryDate(), entries);
 			}
 
 			feed.setLastUpdateSuccess(now);
@@ -98,6 +99,8 @@ public class FeedRefreshWorker {
 			feed.setLastModifiedHeader(fetchedFeed.getFeed()
 					.getLastModifiedHeader());
 			feed.setEtagHeader(fetchedFeed.getFeed().getEtagHeader());
+			feed.setLastPublishedDate(fetchedFeed.getFeed()
+					.getLastPublishedDate());
 
 			feed.setErrorCount(0);
 			feed.setMessage(null);

src/main/java/com/commafeed/backend/feeds/FetchedFeed.java

@@ -14,7 +14,7 @@ public class FetchedFeed {
 
 	private String title;
 	private long fetchDuration;
-	private Date publishedDate;
+	private Date lastEntryDate;
 
 	/**
 	 * pubsubhubbub hub url
@@ -58,14 +58,6 @@ public class FetchedFeed {
 		this.fetchDuration = fetchDuration;
 	}
 
-	public Date getPublishedDate() {
-		return publishedDate;
-	}
-
-	public void setPublishedDate(Date publishedDate) {
-		this.publishedDate = publishedDate;
-	}
-
 	public String getHub() {
 		return hub;
 	}
@@ -82,4 +74,12 @@ public class FetchedFeed {
 		this.topic = topic;
 	}
 
+	public Date getLastEntryDate() {
+		return lastEntryDate;
+	}
+
+	public void setLastEntryDate(Date lastEntryDate) {
+		this.lastEntryDate = lastEntryDate;
+	}
+
 }

src/main/java/com/commafeed/backend/model/Feed.java

@@ -45,6 +45,12 @@ public class Feed extends AbstractModel {
 	@Index(name = "lastupdated_index")
 	private Date lastUpdated;
 
+	/**
+	 * Last publishedDate value in the feed
+	 */
+	@Temporal(TemporalType.TIMESTAMP)
+	private Date lastPublishedDate;
+
 	/**
 	 * Last time we successfully refreshed the feed
 	 */
@@ -210,4 +216,12 @@ public class Feed extends AbstractModel {
 		this.pushLastPing = pushLastPing;
 	}
 
+	public Date getLastPublishedDate() {
+		return lastPublishedDate;
+	}
+
+	public void setLastPublishedDate(Date lastPublishedDate) {
+		this.lastPublishedDate = lastPublishedDate;
+	}
+
 }

src/main/java/com/commafeed/frontend/rest/resources/FeedREST.java

@@ -163,7 +163,7 @@ public class FeedREST extends AbstractResourceREST {
 		url = StringUtils.trimToEmpty(url);
 		url = prependHttp(url);
 		try {
-			FetchedFeed feed = feedFetcher.fetch(url, true, null, null);
+			FetchedFeed feed = feedFetcher.fetch(url, true, null, null, null);
 			info = new FeedInfo();
 			info.setUrl(feed.getFeed().getUrl());
 			info.setTitle(feed.getTitle());