garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

honor Cache-Control response header (#1615)

Browse Source
This commit is contained in:
Athou
2024-11-29 15:32:54 +01:00
parent 477f2cd6db
commit 7a838cddad
5 changed files with 30 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -34,6 +34,7 @@ import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
@@ -46,6 +47,7 @@ import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.CacheControl;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@@ -123,8 +125,13 @@ public class HttpGetter {
throw new NotModifiedException("eTagHeader is the same");
}
Duration validFor = Optional.ofNullable(response.getCacheControl())
.filter(cc -> cc.getMaxAge() >= 0)
.map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO);
return new HttpResult(response.getContent(), response.getContentType(), lastModifiedHeader, eTagHeader,
response.getUrlAfterRedirect());
response.getUrlAfterRedirect(), validFor);
}
private HttpResponse invoke(HttpRequest request) throws IOException {
@@ -152,6 +159,12 @@ public class HttpGetter {
.map(StringUtils::trimToNull)
.orElse(null);
CacheControl cacheControl = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(CacheControlDelegate.INSTANCE::fromString)
.orElse(null);
String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null);
String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations())
.map(RedirectLocations::getAll)
@@ -159,7 +172,7 @@ public class HttpGetter {
.map(URI::toString)
.orElse(request.getUrl());
return new HttpResponse(code, lastModifiedHeader, eTagHeader, content, contentType, urlAfterRedirect);
return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, content, contentType, urlAfterRedirect);
});
}
@@ -311,6 +324,7 @@ public class HttpGetter {
int code;
String lastModifiedHeader;
String eTagHeader;
CacheControl cacheControl;
byte[] content;
String contentType;
String urlAfterRedirect;
@@ -323,6 +337,7 @@ public class HttpGetter {
String lastModifiedSince;
String eTag;
String urlAfterRedirect;
Duration validFor;
}
}

6
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -2,6 +2,7 @@ package com.commafeed.backend.feed;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
@@ -87,7 +88,8 @@ public class FeedFetcher {
etagHeaderValueChanged ? result.getETag() : null);
}
return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash);
return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash,
result.getValidFor());
}
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
@@ -102,7 +104,7 @@ public class FeedFetcher {
}
public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader,
String contentHash) {
String contentHash, Duration validFor) {
}
}

6
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -6,6 +6,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
@@ -76,8 +77,9 @@ public class FeedRefreshWorker {
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(
refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(), result.feed().averageEntryInterval()));
feed.setDisabledUntil(ObjectUtils.max(
refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(), result.feed().averageEntryInterval()),
Instant.now().plus(result.validFor())));
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {

4
commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java
View File

@@ -93,13 +93,15 @@ class HttpGetterTest {
.withBody(feedContent)
.withContentType(MediaType.APPLICATION_ATOM_XML)
.withHeader(HttpHeaders.LAST_MODIFIED, "123456")
.withHeader(HttpHeaders.ETAG, "78910"));
.withHeader(HttpHeaders.ETAG, "78910")
.withHeader(HttpHeaders.CACHE_CONTROL, "max-age=60"));
HttpResult result = getter.get(this.feedUrl);
Assertions.assertArrayEquals(feedContent, result.getContent());
Assertions.assertEquals(MediaType.APPLICATION_ATOM_XML.toString(), result.getContentType());
Assertions.assertEquals("123456", result.getLastModifiedSince());
Assertions.assertEquals("78910", result.getETag());
Assertions.assertEquals(Duration.ofSeconds(60), result.getValidFor());
Assertions.assertEquals(this.feedUrl, result.getUrlAfterRedirect());
}

3
commafeed-server/src/test/java/com/commafeed/backend/feed/FeedFetcherTest.java
View File

@@ -1,5 +1,6 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
@@ -46,7 +47,7 @@ class FeedFetcherTest {
String lastContentHash = Digests.sha1Hex(content);
Mockito.when(getter.get(HttpGetter.HttpRequest.builder(url).lastModified(lastModified).eTag(etag).build()))
.thenReturn(new HttpResult(content, "content-type", "last-modified-2", "etag-2", null));
.thenReturn(new HttpResult(content, "content-type", "last-modified-2", "etag-2", null, Duration.ZERO));
NotModifiedException e = Assertions.assertThrows(NotModifiedException.class,
() -> fetcher.fetch(url, false, lastModified, etag, Instant.now(), lastContentHash));