From 7a838cddade05146bbdb036508818d390eb5f39e Mon Sep 17 00:00:00 2001 From: Athou Date: Fri, 29 Nov 2024 15:32:54 +0100 Subject: [PATCH] honor Cache-Control response header (#1615) --- .../com/commafeed/backend/HttpGetter.java | 19 +++++++++++++++++-- .../commafeed/backend/feed/FeedFetcher.java | 6 ++++-- .../backend/feed/FeedRefreshWorker.java | 6 ++++-- .../com/commafeed/backend/HttpGetterTest.java | 4 +++- .../backend/feed/FeedFetcherTest.java | 3 ++- 5 files changed, 30 insertions(+), 8 deletions(-) diff --git a/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java b/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java index 72bede7a..4a1ab22f 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java @@ -34,6 +34,7 @@ import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.util.TimeValue; import org.apache.hc.core5.util.Timeout; +import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate; import com.codahale.metrics.MetricRegistry; import com.commafeed.CommaFeedConfiguration; @@ -46,6 +47,7 @@ import com.google.common.io.ByteStreams; import com.google.common.net.HttpHeaders; import jakarta.inject.Singleton; +import jakarta.ws.rs.core.CacheControl; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -123,8 +125,13 @@ public class HttpGetter { throw new NotModifiedException("eTagHeader is the same"); } + Duration validFor = Optional.ofNullable(response.getCacheControl()) + .filter(cc -> cc.getMaxAge() >= 0) + .map(cc -> Duration.ofSeconds(cc.getMaxAge())) + .orElse(Duration.ZERO); + return new HttpResult(response.getContent(), response.getContentType(), lastModifiedHeader, eTagHeader, - response.getUrlAfterRedirect()); + response.getUrlAfterRedirect(), validFor); } private HttpResponse invoke(HttpRequest request) throws IOException { @@ -152,6 +159,12 @@ public class HttpGetter { .map(StringUtils::trimToNull) .orElse(null); + CacheControl cacheControl = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL)) + .map(NameValuePair::getValue) + .map(StringUtils::trimToNull) + .map(CacheControlDelegate.INSTANCE::fromString) + .orElse(null); + String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null); String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations()) .map(RedirectLocations::getAll) @@ -159,7 +172,7 @@ public class HttpGetter { .map(URI::toString) .orElse(request.getUrl()); - return new HttpResponse(code, lastModifiedHeader, eTagHeader, content, contentType, urlAfterRedirect); + return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, content, contentType, urlAfterRedirect); }); } @@ -311,6 +324,7 @@ public class HttpGetter { int code; String lastModifiedHeader; String eTagHeader; + CacheControl cacheControl; byte[] content; String contentType; String urlAfterRedirect; @@ -323,6 +337,7 @@ public class HttpGetter { String lastModifiedSince; String eTag; String urlAfterRedirect; + Duration validFor; } } diff --git a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java index 6a8b11f9..6b029d82 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java @@ -2,6 +2,7 @@ package com.commafeed.backend.feed; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.time.Instant; import java.util.List; @@ -87,7 +88,8 @@ public class FeedFetcher { etagHeaderValueChanged ? result.getETag() : null); } - return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash); + return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash, + result.getValidFor()); } private static String extractFeedUrl(List urlProviders, String url, String urlContent) { @@ -102,7 +104,7 @@ public class FeedFetcher { } public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader, - String contentHash) { + String contentHash, Duration validFor) { } } diff --git a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java index a6811d97..e1e6d0dd 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java @@ -6,6 +6,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import com.codahale.metrics.Meter; @@ -76,8 +77,9 @@ public class FeedRefreshWorker { feed.setErrorCount(0); feed.setMessage(null); - feed.setDisabledUntil( - refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(), result.feed().averageEntryInterval())); + feed.setDisabledUntil(ObjectUtils.max( + refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(), result.feed().averageEntryInterval()), + Instant.now().plus(result.validFor()))); return new FeedRefreshWorkerResult(feed, entries); } catch (NotModifiedException e) { diff --git a/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java b/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java index 8a69394c..0244aa7e 100644 --- a/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java +++ b/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java @@ -93,13 +93,15 @@ class HttpGetterTest { .withBody(feedContent) .withContentType(MediaType.APPLICATION_ATOM_XML) .withHeader(HttpHeaders.LAST_MODIFIED, "123456") - .withHeader(HttpHeaders.ETAG, "78910")); + .withHeader(HttpHeaders.ETAG, "78910") + .withHeader(HttpHeaders.CACHE_CONTROL, "max-age=60")); HttpResult result = getter.get(this.feedUrl); Assertions.assertArrayEquals(feedContent, result.getContent()); Assertions.assertEquals(MediaType.APPLICATION_ATOM_XML.toString(), result.getContentType()); Assertions.assertEquals("123456", result.getLastModifiedSince()); Assertions.assertEquals("78910", result.getETag()); + Assertions.assertEquals(Duration.ofSeconds(60), result.getValidFor()); Assertions.assertEquals(this.feedUrl, result.getUrlAfterRedirect()); } diff --git a/commafeed-server/src/test/java/com/commafeed/backend/feed/FeedFetcherTest.java b/commafeed-server/src/test/java/com/commafeed/backend/feed/FeedFetcherTest.java index 03ad1142..eedd7ae8 100644 --- a/commafeed-server/src/test/java/com/commafeed/backend/feed/FeedFetcherTest.java +++ b/commafeed-server/src/test/java/com/commafeed/backend/feed/FeedFetcherTest.java @@ -1,5 +1,6 @@ package com.commafeed.backend.feed; +import java.time.Duration; import java.time.Instant; import java.util.List; @@ -46,7 +47,7 @@ class FeedFetcherTest { String lastContentHash = Digests.sha1Hex(content); Mockito.when(getter.get(HttpGetter.HttpRequest.builder(url).lastModified(lastModified).eTag(etag).build())) - .thenReturn(new HttpResult(content, "content-type", "last-modified-2", "etag-2", null)); + .thenReturn(new HttpResult(content, "content-type", "last-modified-2", "etag-2", null, Duration.ZERO)); NotModifiedException e = Assertions.assertThrows(NotModifiedException.class, () -> fetcher.fetch(url, false, lastModified, etag, Instant.now(), lastContentHash));