add support for the Retry-After header sent by OpenRSS

commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java

@@ -1,5 +1,7 @@
 package com.commafeed;
 
+import java.time.InstantSource;
+
 import com.codahale.metrics.MetricRegistry;
 
 import jakarta.enterprise.inject.Produces;
@@ -8,9 +10,16 @@ import jakarta.inject.Singleton;
 @Singleton
 public class CommaFeedProducers {
 
+	@Produces
+	@Singleton
+	public InstantSource instantSource() {
+		return InstantSource.system();
+	}
+
 	@Produces
 	@Singleton
 	public MetricRegistry metricRegistry() {
 		return new MetricRegistry();
 	}
+
 }

commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java

@@ -7,9 +7,12 @@ import java.net.InetAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
 import java.time.Duration;
+import java.time.Instant;
+import java.time.InstantSource;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 import org.apache.commons.lang3.StringUtils;
@@ -25,6 +28,7 @@ import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuil
 import org.apache.hc.client5.http.io.HttpClientConnectionManager;
 import org.apache.hc.client5.http.protocol.HttpClientContext;
 import org.apache.hc.client5.http.protocol.RedirectLocations;
+import org.apache.hc.client5.http.utils.DateUtils;
 import org.apache.hc.core5.http.ClassicHttpRequest;
 import org.apache.hc.core5.http.Header;
 import org.apache.hc.core5.http.HttpEntity;
@@ -51,6 +55,7 @@ import jakarta.ws.rs.core.CacheControl;
 import lombok.Builder;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.RequiredArgsConstructor;
 import lombok.Value;
 import lombok.extern.slf4j.Slf4j;
 import nl.altindag.ssl.SSLFactory;
@@ -64,11 +69,13 @@ import nl.altindag.ssl.apache5.util.Apache5SslUtils;
 public class HttpGetter {
 
 	private final CommaFeedConfiguration config;
+	private final InstantSource instantSource;
 	private final CloseableHttpClient client;
 	private final Cache<HttpRequest, HttpResponse> cache;
 
-	public HttpGetter(CommaFeedConfiguration config, CommaFeedVersion version, MetricRegistry metrics) {
+	public HttpGetter(CommaFeedConfiguration config, InstantSource instantSource, CommaFeedVersion version, MetricRegistry metrics) {
 		this.config = config;
+		this.instantSource = instantSource;
 
 		PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config);
 		String userAgent = config.httpClient()
@@ -88,11 +95,11 @@ public class HttpGetter {
 				() -> cache == null ? 0 : cache.asMap().values().stream().mapToInt(e -> e.content != null ? e.content.length : 0).sum());
 	}
 
-	public HttpResult get(String url) throws IOException, NotModifiedException {
+	public HttpResult get(String url) throws IOException, NotModifiedException, TooManyRequestsException {
 		return get(HttpRequest.builder(url).build());
 	}
 
-	public HttpResult get(HttpRequest request) throws IOException, NotModifiedException {
+	public HttpResult get(HttpRequest request) throws IOException, NotModifiedException, TooManyRequestsException {
 		final HttpResponse response;
 		if (cache == null) {
 			response = invoke(request);
@@ -109,9 +116,15 @@ public class HttpGetter {
 		}
 
 		int code = response.getCode();
+		if (Set.of(HttpStatus.SC_TOO_MANY_REQUESTS, HttpStatus.SC_SERVICE_UNAVAILABLE).contains(code) && response.getRetryAfter() != null) {
+			throw new TooManyRequestsException(response.getRetryAfter());
+		}
+
 		if (code == HttpStatus.SC_NOT_MODIFIED) {
 			throw new NotModifiedException("'304 - not modified' http code received");
-		} else if (code >= 300) {
+		}
+
+		if (code >= 300) {
 			throw new HttpResponseException(code, "Server returned HTTP error code " + code);
 		}
 
@@ -165,6 +178,12 @@ public class HttpGetter {
 					.map(HttpGetter::toCacheControl)
 					.orElse(null);
 
+			Instant retryAfter = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
+					.map(NameValuePair::getValue)
+					.map(StringUtils::trimToNull)
+					.map(this::toInstant)
+					.orElse(null);
+
 			String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null);
 			String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations())
 					.map(RedirectLocations::getAll)
@@ -172,7 +191,7 @@ public class HttpGetter {
 					.map(URI::toString)
 					.orElse(request.getUrl());
 
-			return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, content, contentType, urlAfterRedirect);
+			return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, retryAfter, content, contentType, urlAfterRedirect);
 		});
 	}
 
@@ -185,6 +204,18 @@ public class HttpGetter {
 		}
 	}
 
+	private Instant toInstant(String headerValue) {
+		if (headerValue == null) {
+			return null;
+		}
+
+		if (StringUtils.isNumeric(headerValue)) {
+			return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
+		}
+
+		return DateUtils.parseStandardDate(headerValue);
+	}
+
 	private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
 		if (entity.getContentLength() > maxBytes) {
 			throw new IOException(
@@ -292,6 +323,14 @@ public class HttpGetter {
 		}
 	}
 
+	@RequiredArgsConstructor
+	@Getter
+	public static class TooManyRequestsException extends Exception {
+		private static final long serialVersionUID = 1L;
+
+		private final Instant retryAfter;
+	}
+
 	@Getter
 	public static class HttpResponseException extends IOException {
 		private static final long serialVersionUID = 1L;
@@ -334,6 +373,7 @@ public class HttpGetter {
 		String lastModifiedHeader;
 		String eTagHeader;
 		CacheControl cacheControl;
+		Instant retryAfter;
 		byte[] content;
 		String contentType;
 		String urlAfterRedirect;

commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java

@@ -13,6 +13,7 @@ import com.commafeed.CommaFeedConfiguration;
 import com.commafeed.backend.HttpGetter;
 import com.commafeed.backend.HttpGetter.HttpResult;
 import com.commafeed.backend.HttpGetter.NotModifiedException;
+import com.commafeed.backend.HttpGetter.TooManyRequestsException;
 import com.commafeed.backend.model.Feed;
 import com.fasterxml.jackson.core.JsonPointer;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -91,7 +92,7 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
 		return new Favicon(bytes, contentType);
 	}
 
-	private byte[] fetchForUser(String googleAuthKey, String userId) throws IOException, NotModifiedException {
+	private byte[] fetchForUser(String googleAuthKey, String userId) throws IOException, NotModifiedException, TooManyRequestsException {
 		URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
 				.queryParam("part", "snippet")
 				.queryParam("key", googleAuthKey)
@@ -100,7 +101,8 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
 		return getter.get(uri.toString()).getContent();
 	}
 
-	private byte[] fetchForChannel(String googleAuthKey, String channelId) throws IOException, NotModifiedException {
+	private byte[] fetchForChannel(String googleAuthKey, String channelId)
+			throws IOException, NotModifiedException, TooManyRequestsException {
 		URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
 				.queryParam("part", "snippet")
 				.queryParam("key", googleAuthKey)
@@ -109,7 +111,8 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
 		return getter.get(uri.toString()).getContent();
 	}
 
-	private byte[] fetchForPlaylist(String googleAuthKey, String playlistId) throws IOException, NotModifiedException {
+	private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
+			throws IOException, NotModifiedException, TooManyRequestsException {
 		URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
 				.queryParam("part", "snippet")
 				.queryParam("key", googleAuthKey)

commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java

@@ -13,6 +13,7 @@ import com.commafeed.backend.HttpGetter;
 import com.commafeed.backend.HttpGetter.HttpRequest;
 import com.commafeed.backend.HttpGetter.HttpResult;
 import com.commafeed.backend.HttpGetter.NotModifiedException;
+import com.commafeed.backend.HttpGetter.TooManyRequestsException;
 import com.commafeed.backend.feed.parser.FeedParser;
 import com.commafeed.backend.feed.parser.FeedParserResult;
 import com.commafeed.backend.urlprovider.FeedURLProvider;
@@ -40,7 +41,8 @@ public class FeedFetcher {
 	}
 
 	public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
-			Instant lastPublishedDate, String lastContentHash) throws FeedException, IOException, NotModifiedException {
+			Instant lastPublishedDate, String lastContentHash)
+			throws FeedException, IOException, NotModifiedException, TooManyRequestsException {
 		log.debug("Fetching feed {}", feedUrl);
 
 		HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());

commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java

@@ -13,6 +13,7 @@ import com.codahale.metrics.Meter;
 import com.codahale.metrics.MetricRegistry;
 import com.commafeed.CommaFeedConfiguration;
 import com.commafeed.backend.HttpGetter.NotModifiedException;
+import com.commafeed.backend.HttpGetter.TooManyRequestsException;
 import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
 import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
 import com.commafeed.backend.model.Feed;
@@ -97,6 +98,14 @@ public class FeedRefreshWorker {
 				feed.setEtagHeader(e.getNewEtagHeader());
 			}
 
+			return new FeedRefreshWorkerResult(feed, Collections.emptyList());
+		} catch (TooManyRequestsException e) {
+			log.debug("Too many requests : {}", feed.getUrl());
+
+			feed.setErrorCount(feed.getErrorCount() + 1);
+			feed.setMessage("Server indicated that we are sending too many requests");
+			feed.setDisabledUntil(e.getRetryAfter());
+
 			return new FeedRefreshWorkerResult(feed, Collections.emptyList());
 		} catch (Exception e) {
 			log.debug("unable to refresh feed {}", feed.getUrl(), e);

commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java

@@ -6,6 +6,7 @@ import java.io.OutputStream;
 import java.math.BigInteger;
 import java.net.SocketTimeoutException;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Objects;
 import java.util.Optional;
@@ -39,6 +40,7 @@ import com.commafeed.CommaFeedVersion;
 import com.commafeed.backend.HttpGetter.HttpResponseException;
 import com.commafeed.backend.HttpGetter.HttpResult;
 import com.commafeed.backend.HttpGetter.NotModifiedException;
+import com.commafeed.backend.HttpGetter.TooManyRequestsException;
 import com.google.common.net.HttpHeaders;
 
 import io.quarkus.runtime.configuration.MemorySize;
@@ -46,9 +48,12 @@ import io.quarkus.runtime.configuration.MemorySize;
 @ExtendWith(MockServerExtension.class)
 class HttpGetterTest {
 
+	private static final Instant NOW = Instant.now();
+
 	private MockServerClient mockServerClient;
 	private String feedUrl;
 	private byte[] feedContent;
+
 	private CommaFeedConfiguration config;
 
 	private HttpGetter getter;
@@ -73,7 +78,7 @@ class HttpGetterTest {
 		Mockito.when(config.httpClient().cache().expiration()).thenReturn(Duration.ofMinutes(1));
 		Mockito.when(config.feedRefresh().httpThreads()).thenReturn(3);
 
-		this.getter = new HttpGetter(config, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class));
+		this.getter = new HttpGetter(config, () -> NOW, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class));
 	}
 
 	@ParameterizedTest
@@ -94,7 +99,8 @@ class HttpGetterTest {
 						.withContentType(MediaType.APPLICATION_ATOM_XML)
 						.withHeader(HttpHeaders.LAST_MODIFIED, "123456")
 						.withHeader(HttpHeaders.ETAG, "78910")
-						.withHeader(HttpHeaders.CACHE_CONTROL, "max-age=60, must-revalidate"));
+						.withHeader(HttpHeaders.CACHE_CONTROL, "max-age=60, must-revalidate")
+						.withHeader(HttpHeaders.RETRY_AFTER, "120"));
 
 		HttpResult result = getter.get(this.feedUrl);
 		Assertions.assertArrayEquals(feedContent, result.getContent());
@@ -117,6 +123,27 @@ class HttpGetterTest {
 		Assertions.assertEquals(Duration.ZERO, result.getValidFor());
 	}
 
+	@Test
+	void tooManyRequestsExceptionSeconds() {
+		this.mockServerClient.when(HttpRequest.request().withMethod("GET"))
+				.respond(
+						HttpResponse.response().withStatusCode(HttpStatus.SC_TOO_MANY_REQUESTS).withHeader(HttpHeaders.RETRY_AFTER, "120"));
+
+		TooManyRequestsException e = Assertions.assertThrows(TooManyRequestsException.class, () -> getter.get(this.feedUrl));
+		Assertions.assertEquals(NOW.plusSeconds(120), e.getRetryAfter());
+	}
+
+	@Test
+	void tooManyRequestsExceptionDate() {
+		this.mockServerClient.when(HttpRequest.request().withMethod("GET"))
+				.respond(HttpResponse.response()
+						.withStatusCode(HttpStatus.SC_TOO_MANY_REQUESTS)
+						.withHeader(HttpHeaders.RETRY_AFTER, "Wed, 21 Oct 2015 07:28:00 GMT"));
+
+		TooManyRequestsException e = Assertions.assertThrows(TooManyRequestsException.class, () -> getter.get(this.feedUrl));
+		Assertions.assertEquals(Instant.parse("2015-10-21T07:28:00Z"), e.getRetryAfter());
+	}
+
 	@ParameterizedTest
 	@ValueSource(
 			ints = { HttpStatus.SC_MOVED_PERMANENTLY, HttpStatus.SC_MOVED_TEMPORARILY, HttpStatus.SC_TEMPORARY_REDIRECT,
@@ -145,7 +172,7 @@ class HttpGetterTest {
 	@Test
 	void dataTimeout() {
 		Mockito.when(config.httpClient().responseTimeout()).thenReturn(Duration.ofMillis(500));
-		this.getter = new HttpGetter(config, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class));
+		this.getter = new HttpGetter(config, () -> NOW, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class));
 
 		this.mockServerClient.when(HttpRequest.request().withMethod("GET"))
 				.respond(HttpResponse.response().withDelay(Delay.milliseconds(1000)));
@@ -156,7 +183,7 @@ class HttpGetterTest {
 	@Test
 	void connectTimeout() {
 		Mockito.when(config.httpClient().connectTimeout()).thenReturn(Duration.ofMillis(500));
-		this.getter = new HttpGetter(config, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class));
+		this.getter = new HttpGetter(config, () -> NOW, Mockito.mock(CommaFeedVersion.class), Mockito.mock(MetricRegistry.class));
 		// try to connect to a non-routable address
 		// https://stackoverflow.com/a/904609
 		Assertions.assertThrows(ConnectTimeoutException.class, () -> getter.get("http://10.255.255.1"));
@@ -209,7 +236,7 @@ class HttpGetterTest {
 	}
 
 	@Test
-	void cacheSubsequentCalls() throws IOException, NotModifiedException {
+	void cacheSubsequentCalls() throws IOException, NotModifiedException, TooManyRequestsException {
 		AtomicInteger calls = new AtomicInteger();
 
 		this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(req -> {
@@ -275,17 +302,17 @@ class HttpGetterTest {
 	class Compression {
 
 		@Test
-		void deflate() throws IOException, NotModifiedException {
+		void deflate() throws IOException, NotModifiedException, TooManyRequestsException {
 			supportsCompression("deflate", DeflaterOutputStream::new);
 		}
 
 		@Test
-		void gzip() throws IOException, NotModifiedException {
+		void gzip() throws IOException, NotModifiedException, TooManyRequestsException {
 			supportsCompression("gzip", GZIPOutputStream::new);
 		}
 
 		void supportsCompression(String encoding, CompressionOutputStreamFunction compressionOutputStreamFunction)
-				throws IOException, NotModifiedException {
+				throws IOException, NotModifiedException, TooManyRequestsException {
 			String body = "my body";
 
 			HttpGetterTest.this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(req -> {