diff --git a/commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java b/commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java index 7ab50216..5fb10c10 100644 --- a/commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java +++ b/commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java @@ -1,5 +1,7 @@ package com.commafeed; +import java.time.InstantSource; + import com.codahale.metrics.MetricRegistry; import jakarta.enterprise.inject.Produces; @@ -8,9 +10,16 @@ import jakarta.inject.Singleton; @Singleton public class CommaFeedProducers { + @Produces + @Singleton + public InstantSource instantSource() { + return InstantSource.system(); + } + @Produces @Singleton public MetricRegistry metricRegistry() { return new MetricRegistry(); } + } diff --git a/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java b/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java index d3118fc2..1b4ccb88 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java @@ -7,9 +7,12 @@ import java.net.InetAddress; import java.net.URI; import java.net.UnknownHostException; import java.time.Duration; +import java.time.Instant; +import java.time.InstantSource; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutionException; import org.apache.commons.lang3.StringUtils; @@ -25,6 +28,7 @@ import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuil import org.apache.hc.client5.http.io.HttpClientConnectionManager; import org.apache.hc.client5.http.protocol.HttpClientContext; import org.apache.hc.client5.http.protocol.RedirectLocations; +import org.apache.hc.client5.http.utils.DateUtils; import org.apache.hc.core5.http.ClassicHttpRequest; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpEntity; @@ -51,6 +55,7 @@ import jakarta.ws.rs.core.CacheControl; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.Value; import lombok.extern.slf4j.Slf4j; import nl.altindag.ssl.SSLFactory; @@ -64,11 +69,13 @@ import nl.altindag.ssl.apache5.util.Apache5SslUtils; public class HttpGetter { private final CommaFeedConfiguration config; + private final InstantSource instantSource; private final CloseableHttpClient client; private final Cache cache; - public HttpGetter(CommaFeedConfiguration config, CommaFeedVersion version, MetricRegistry metrics) { + public HttpGetter(CommaFeedConfiguration config, InstantSource instantSource, CommaFeedVersion version, MetricRegistry metrics) { this.config = config; + this.instantSource = instantSource; PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config); String userAgent = config.httpClient() @@ -88,11 +95,11 @@ public class HttpGetter { () -> cache == null ? 0 : cache.asMap().values().stream().mapToInt(e -> e.content != null ? e.content.length : 0).sum()); } - public HttpResult get(String url) throws IOException, NotModifiedException { + public HttpResult get(String url) throws IOException, NotModifiedException, TooManyRequestsException { return get(HttpRequest.builder(url).build()); } - public HttpResult get(HttpRequest request) throws IOException, NotModifiedException { + public HttpResult get(HttpRequest request) throws IOException, NotModifiedException, TooManyRequestsException { final HttpResponse response; if (cache == null) { response = invoke(request); @@ -109,9 +116,15 @@ public class HttpGetter { } int code = response.getCode(); + if (Set.of(HttpStatus.SC_TOO_MANY_REQUESTS, HttpStatus.SC_SERVICE_UNAVAILABLE).contains(code) && response.getRetryAfter() != null) { + throw new TooManyRequestsException(response.getRetryAfter()); + } + if (code == HttpStatus.SC_NOT_MODIFIED) { throw new NotModifiedException("'304 - not modified' http code received"); - } else if (code >= 300) { + } + + if (code >= 300) { throw new HttpResponseException(code, "Server returned HTTP error code " + code); } @@ -165,6 +178,12 @@ public class HttpGetter { .map(HttpGetter::toCacheControl) .orElse(null); + Instant retryAfter = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER)) + .map(NameValuePair::getValue) + .map(StringUtils::trimToNull) + .map(this::toInstant) + .orElse(null); + String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null); String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations()) .map(RedirectLocations::getAll) @@ -172,7 +191,7 @@ public class HttpGetter { .map(URI::toString) .orElse(request.getUrl()); - return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, content, contentType, urlAfterRedirect); + return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, retryAfter, content, contentType, urlAfterRedirect); }); } @@ -185,6 +204,18 @@ public class HttpGetter { } } + private Instant toInstant(String headerValue) { + if (headerValue == null) { + return null; + } + + if (StringUtils.isNumeric(headerValue)) { + return instantSource.instant().plusSeconds(Long.parseLong(headerValue)); + } + + return DateUtils.parseStandardDate(headerValue); + } + private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException { if (entity.getContentLength() > maxBytes) { throw new IOException( @@ -292,6 +323,14 @@ public class HttpGetter { } } + @RequiredArgsConstructor + @Getter + public static class TooManyRequestsException extends Exception { + private static final long serialVersionUID = 1L; + + private final Instant retryAfter; + } + @Getter public static class HttpResponseException extends IOException { private static final long serialVersionUID = 1L; @@ -334,6 +373,7 @@ public class HttpGetter { String lastModifiedHeader; String eTagHeader; CacheControl cacheControl; + Instant retryAfter; byte[] content; String contentType; String urlAfterRedirect; diff --git a/commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java b/commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java index 86198d04..42cedd85 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java @@ -13,6 +13,7 @@ import com.commafeed.CommaFeedConfiguration; import com.commafeed.backend.HttpGetter; import com.commafeed.backend.HttpGetter.HttpResult; import com.commafeed.backend.HttpGetter.NotModifiedException; +import com.commafeed.backend.HttpGetter.TooManyRequestsException; import com.commafeed.backend.model.Feed; import com.fasterxml.jackson.core.JsonPointer; import com.fasterxml.jackson.databind.JsonNode; @@ -91,7 +92,7 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher { return new Favicon(bytes, contentType); } - private byte[] fetchForUser(String googleAuthKey, String userId) throws IOException, NotModifiedException { + private byte[] fetchForUser(String googleAuthKey, String userId) throws IOException, NotModifiedException, TooManyRequestsException { URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels") .queryParam("part", "snippet") .queryParam("key", googleAuthKey) @@ -100,7 +101,8 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher { return getter.get(uri.toString()).getContent(); } - private byte[] fetchForChannel(String googleAuthKey, String channelId) throws IOException, NotModifiedException { + private byte[] fetchForChannel(String googleAuthKey, String channelId) + throws IOException, NotModifiedException, TooManyRequestsException { URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels") .queryParam("part", "snippet") .queryParam("key", googleAuthKey) @@ -109,7 +111,8 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher { return getter.get(uri.toString()).getContent(); } - private byte[] fetchForPlaylist(String googleAuthKey, String playlistId) throws IOException, NotModifiedException { + private byte[] fetchForPlaylist(String googleAuthKey, String playlistId) + throws IOException, NotModifiedException, TooManyRequestsException { URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists") .queryParam("part", "snippet") .queryParam("key", googleAuthKey) diff --git a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java index 6b029d82..1d88f47a 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java @@ -13,6 +13,7 @@ import com.commafeed.backend.HttpGetter; import com.commafeed.backend.HttpGetter.HttpRequest; import com.commafeed.backend.HttpGetter.HttpResult; import com.commafeed.backend.HttpGetter.NotModifiedException; +import com.commafeed.backend.HttpGetter.TooManyRequestsException; import com.commafeed.backend.feed.parser.FeedParser; import com.commafeed.backend.feed.parser.FeedParserResult; import com.commafeed.backend.urlprovider.FeedURLProvider; @@ -40,7 +41,8 @@ public class FeedFetcher { } public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag, - Instant lastPublishedDate, String lastContentHash) throws FeedException, IOException, NotModifiedException { + Instant lastPublishedDate, String lastContentHash) + throws FeedException, IOException, NotModifiedException, TooManyRequestsException { log.debug("Fetching feed {}", feedUrl); HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build()); diff --git a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java index e1e6d0dd..a6533606 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java @@ -13,6 +13,7 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.commafeed.CommaFeedConfiguration; import com.commafeed.backend.HttpGetter.NotModifiedException; +import com.commafeed.backend.HttpGetter.TooManyRequestsException; import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult; import com.commafeed.backend.feed.parser.FeedParserResult.Entry; import com.commafeed.backend.model.Feed; @@ -97,6 +98,14 @@ public class FeedRefreshWorker { feed.setEtagHeader(e.getNewEtagHeader()); } + return new FeedRefreshWorkerResult(feed, Collections.emptyList()); + } catch (TooManyRequestsException e) { + log.debug("Too many requests : {}", feed.getUrl()); + + feed.setErrorCount(feed.getErrorCount() + 1); + feed.setMessage("Server indicated that we are sending too many requests"); + feed.setDisabledUntil(e.getRetryAfter()); + return new FeedRefreshWorkerResult(feed, Collections.emptyList()); } catch (Exception e) { log.debug("unable to refresh feed {}", feed.getUrl(), e); diff --git a/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java b/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java index f7848343..ededc315 100644 --- a/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java +++ b/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java @@ -6,6 +6,7 @@ import java.io.OutputStream; import java.math.BigInteger; import java.net.SocketTimeoutException; import java.time.Duration; +import java.time.Instant; import java.util.Arrays; import java.util.Objects; import java.util.Optional; @@ -39,6 +40,7 @@ import com.commafeed.CommaFeedVersion; import com.commafeed.backend.HttpGetter.HttpResponseException; import com.commafeed.backend.HttpGetter.HttpResult; import com.commafeed.backend.HttpGetter.NotModifiedException; +import com.commafeed.backend.HttpGetter.TooManyRequestsException; import com.google.common.net.HttpHeaders; import io.quarkus.runtime.configuration.MemorySize; @@ -46,9 +48,12 @@ import io.quarkus.runtime.configuration.MemorySize; @ExtendWith(MockServerExtension.class) class HttpGetterTest { + private static final Instant NOW = Instant.now(); + private MockServerClient mockServerClient; private String feedUrl; private byte[] feedContent; + private CommaFeedConfiguration config; private HttpGetter getter; @@ -73,7 +78,7 @@ class HttpGetterTest { Mockito.when(config.httpClient().cache().expiration()).thenReturn(Duration.ofMinutes(1)); Mockito.when(config.feedRefresh().httpThreads()).thenReturn(3); - this.getter = new HttpGetter(config, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class)); + this.getter = new HttpGetter(config, () -> NOW, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class)); } @ParameterizedTest @@ -94,7 +99,8 @@ class HttpGetterTest { .withContentType(MediaType.APPLICATION_ATOM_XML) .withHeader(HttpHeaders.LAST_MODIFIED, "123456") .withHeader(HttpHeaders.ETAG, "78910") - .withHeader(HttpHeaders.CACHE_CONTROL, "max-age=60, must-revalidate")); + .withHeader(HttpHeaders.CACHE_CONTROL, "max-age=60, must-revalidate") + .withHeader(HttpHeaders.RETRY_AFTER, "120")); HttpResult result = getter.get(this.feedUrl); Assertions.assertArrayEquals(feedContent, result.getContent()); @@ -117,6 +123,27 @@ class HttpGetterTest { Assertions.assertEquals(Duration.ZERO, result.getValidFor()); } + @Test + void tooManyRequestsExceptionSeconds() { + this.mockServerClient.when(HttpRequest.request().withMethod("GET")) + .respond( + HttpResponse.response().withStatusCode(HttpStatus.SC_TOO_MANY_REQUESTS).withHeader(HttpHeaders.RETRY_AFTER, "120")); + + TooManyRequestsException e = Assertions.assertThrows(TooManyRequestsException.class, () -> getter.get(this.feedUrl)); + Assertions.assertEquals(NOW.plusSeconds(120), e.getRetryAfter()); + } + + @Test + void tooManyRequestsExceptionDate() { + this.mockServerClient.when(HttpRequest.request().withMethod("GET")) + .respond(HttpResponse.response() + .withStatusCode(HttpStatus.SC_TOO_MANY_REQUESTS) + .withHeader(HttpHeaders.RETRY_AFTER, "Wed, 21 Oct 2015 07:28:00 GMT")); + + TooManyRequestsException e = Assertions.assertThrows(TooManyRequestsException.class, () -> getter.get(this.feedUrl)); + Assertions.assertEquals(Instant.parse("2015-10-21T07:28:00Z"), e.getRetryAfter()); + } + @ParameterizedTest @ValueSource( ints = { HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY, HttpStatus.SC_TEMPORARY_REDIRECT, @@ -145,7 +172,7 @@ class HttpGetterTest { @Test void dataTimeout() { Mockito.when(config.httpClient().responseTimeout()).thenReturn(Duration.ofMillis(500)); - this.getter = new HttpGetter(config, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class)); + this.getter = new HttpGetter(config, () -> NOW, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class)); this.mockServerClient.when(HttpRequest.request().withMethod("GET")) .respond(HttpResponse.response().withDelay(Delay.milliseconds(1000))); @@ -156,7 +183,7 @@ class HttpGetterTest { @Test void connectTimeout() { Mockito.when(config.httpClient().connectTimeout()).thenReturn(Duration.ofMillis(500)); - this.getter = new HttpGetter(config, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class)); + this.getter = new HttpGetter(config, () -> NOW, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class)); // try to connect to a non-routable address // https://stackoverflow.com/a/904609 Assertions.assertThrows(ConnectTimeoutException.class, () -> getter.get("http://10.255.255.1")); @@ -209,7 +236,7 @@ class HttpGetterTest { } @Test - void cacheSubsequentCalls() throws IOException, NotModifiedException { + void cacheSubsequentCalls() throws IOException, NotModifiedException, TooManyRequestsException { AtomicInteger calls = new AtomicInteger(); this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(req -> { @@ -275,17 +302,17 @@ class HttpGetterTest { class Compression { @Test - void deflate() throws IOException, NotModifiedException { + void deflate() throws IOException, NotModifiedException, TooManyRequestsException { supportsCompression("deflate", DeflaterOutputStream::new); } @Test - void gzip() throws IOException, NotModifiedException { + void gzip() throws IOException, NotModifiedException, TooManyRequestsException { supportsCompression("gzip", GZIPOutputStream::new); } void supportsCompression(String encoding, CompressionOutputStreamFunction compressionOutputStreamFunction) - throws IOException, NotModifiedException { + throws IOException, NotModifiedException, TooManyRequestsException { String body = "my body"; HttpGetterTest.this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(req -> {