diff --git a/src/main/java/com/commafeed/backend/feeds/FeedFetcher.java b/src/main/java/com/commafeed/backend/feeds/FeedFetcher.java index 763ffd23..502db2ad 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedFetcher.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedFetcher.java @@ -1,6 +1,7 @@ package com.commafeed.backend.feeds; import java.io.IOException; +import java.util.Date; import javax.inject.Inject; @@ -29,8 +30,9 @@ public class FeedFetcher { HttpGetter getter; public FetchedFeed fetch(String feedUrl, boolean extractFeedUrlFromHtml, - String lastModified, String eTag) throws FeedException, - ClientProtocolException, IOException, NotModifiedException { + String lastModified, String eTag, Date lastPublishedDate) + throws FeedException, ClientProtocolException, IOException, + NotModifiedException { log.debug("Fetching feed {}", feedUrl); FetchedFeed fetchedFeed = null; @@ -48,6 +50,15 @@ public class FeedFetcher { } fetchedFeed = parser.parse(feedUrl, result.getContent()); + + if (lastPublishedDate != null + && fetchedFeed.getFeed().getLastPublishedDate() != null + && lastPublishedDate.getTime() == fetchedFeed.getFeed() + .getLastPublishedDate().getTime()) { + log.info("using publishedDate!"); + throw new NotModifiedException(); + } + Feed feed = fetchedFeed.getFeed(); feed.setLastModifiedHeader(result.getLastModifiedSince()); feed.setEtagHeader(FeedUtils.truncate(result.geteTag(), 255)); diff --git a/src/main/java/com/commafeed/backend/feeds/FeedParser.java b/src/main/java/com/commafeed/backend/feeds/FeedParser.java index 43cff29d..2b5e54d0 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedParser.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedParser.java @@ -71,6 +71,7 @@ public class FeedParser { fetchedFeed.setTopic(findSelf(rss)); feed.setUrl(feedUrl); feed.setLink(rss.getLink()); + feed.setLastPublishedDate(rss.getPublishedDate()); List items = rss.getEntries(); for (SyndEntry item : items) { FeedEntry entry = new FeedEntry(); @@ -105,12 +106,12 @@ public class FeedParser { entries.add(entry); } - Date publishedDate = null; + Date lastEntryDate = null; if (!entries.isEmpty()) { Long timestamp = FeedUtils.getSortedTimestamps(entries).get(0); - publishedDate = new Date(timestamp); + lastEntryDate = new Date(timestamp); } - fetchedFeed.setPublishedDate(publishedDate); + fetchedFeed.setLastEntryDate(lastEntryDate); } catch (Exception e) { throw new FeedException(String.format( diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java index bcd60356..14c2f73f 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshWorker.java @@ -82,7 +82,8 @@ public class FeedRefreshWorker { Date now = Calendar.getInstance().getTime(); try { FetchedFeed fetchedFeed = fetcher.fetch(feed.getUrl(), false, - feed.getLastModifiedHeader(), feed.getEtagHeader()); + feed.getLastModifiedHeader(), feed.getEtagHeader(), + feed.getLastPublishedDate()); // stops here if NotModifiedException or any other exception is // thrown List entries = fetchedFeed.getEntries(); @@ -90,7 +91,7 @@ public class FeedRefreshWorker { Date disabledUntil = null; if (applicationSettingsService.get().isHeavyLoad()) { disabledUntil = FeedUtils.buildDisabledUntil( - fetchedFeed.getPublishedDate(), entries); + fetchedFeed.getLastEntryDate(), entries); } feed.setLastUpdateSuccess(now); @@ -98,6 +99,8 @@ public class FeedRefreshWorker { feed.setLastModifiedHeader(fetchedFeed.getFeed() .getLastModifiedHeader()); feed.setEtagHeader(fetchedFeed.getFeed().getEtagHeader()); + feed.setLastPublishedDate(fetchedFeed.getFeed() + .getLastPublishedDate()); feed.setErrorCount(0); feed.setMessage(null); diff --git a/src/main/java/com/commafeed/backend/feeds/FetchedFeed.java b/src/main/java/com/commafeed/backend/feeds/FetchedFeed.java index 829323eb..45f72bfb 100644 --- a/src/main/java/com/commafeed/backend/feeds/FetchedFeed.java +++ b/src/main/java/com/commafeed/backend/feeds/FetchedFeed.java @@ -14,7 +14,7 @@ public class FetchedFeed { private String title; private long fetchDuration; - private Date publishedDate; + private Date lastEntryDate; /** * pubsubhubbub hub url @@ -58,14 +58,6 @@ public class FetchedFeed { this.fetchDuration = fetchDuration; } - public Date getPublishedDate() { - return publishedDate; - } - - public void setPublishedDate(Date publishedDate) { - this.publishedDate = publishedDate; - } - public String getHub() { return hub; } @@ -82,4 +74,12 @@ public class FetchedFeed { this.topic = topic; } + public Date getLastEntryDate() { + return lastEntryDate; + } + + public void setLastEntryDate(Date lastEntryDate) { + this.lastEntryDate = lastEntryDate; + } + } diff --git a/src/main/java/com/commafeed/backend/model/Feed.java b/src/main/java/com/commafeed/backend/model/Feed.java index d059d017..633c8717 100644 --- a/src/main/java/com/commafeed/backend/model/Feed.java +++ b/src/main/java/com/commafeed/backend/model/Feed.java @@ -45,6 +45,12 @@ public class Feed extends AbstractModel { @Index(name = "lastupdated_index") private Date lastUpdated; + /** + * Last publishedDate value in the feed + */ + @Temporal(TemporalType.TIMESTAMP) + private Date lastPublishedDate; + /** * Last time we successfully refreshed the feed */ @@ -210,4 +216,12 @@ public class Feed extends AbstractModel { this.pushLastPing = pushLastPing; } + public Date getLastPublishedDate() { + return lastPublishedDate; + } + + public void setLastPublishedDate(Date lastPublishedDate) { + this.lastPublishedDate = lastPublishedDate; + } + } diff --git a/src/main/java/com/commafeed/frontend/rest/resources/FeedREST.java b/src/main/java/com/commafeed/frontend/rest/resources/FeedREST.java index 60bdf168..81d83d01 100644 --- a/src/main/java/com/commafeed/frontend/rest/resources/FeedREST.java +++ b/src/main/java/com/commafeed/frontend/rest/resources/FeedREST.java @@ -163,7 +163,7 @@ public class FeedREST extends AbstractResourceREST { url = StringUtils.trimToEmpty(url); url = prependHttp(url); try { - FetchedFeed feed = feedFetcher.fetch(url, true, null, null); + FetchedFeed feed = feedFetcher.fetch(url, true, null, null, null); info = new FeedInfo(); info.setUrl(feed.getFeed().getUrl()); info.setTitle(feed.getTitle()); diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index f54e01c5..4c3147da 100644 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -1,9 +1,7 @@ log4j.logger.com.commafeed=INFO, CONSOLE log4j.logger.org=WARN, CONSOLE -log4j.logger.com=WARN, CONSOLE log4j.logger.ro=WARN, CONSOLE -log4j.logger.net.sf.ehcache=ALL, CONSOLE -log4j.logger.org.hibernate.cache=ALL, CONSOLE +log4j.logger.com.wordnik=WARN, CONSOLE log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout diff --git a/src/main/webapp/js/controllers.js b/src/main/webapp/js/controllers.js index 94968ec8..9a944f68 100644 --- a/src/main/webapp/js/controllers.js +++ b/src/main/webapp/js/controllers.js @@ -482,11 +482,6 @@ function($scope, $http, $state, $stateParams, $route, $location, }; $scope.refresh = function() { - if($stateParams._type == 'feed'){ - FeedService.refresh({ - id : $stateParams._id - }); - } $scope.$emit('emitReload'); }; @@ -998,6 +993,12 @@ function($scope, $stateParams, $http, $route, $window, EntryService, SettingsSer $scope.busy = false; $scope.hasMore = true; $scope.loadMoreEntries(); + + if ($scope.selectedType == 'feed'){ + FeedService.refresh({ + id : $stateParams._id + }); + } }); }]);