Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-03-21 21:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

add support for the Retry-After header sent by OpenRSS

Browse Source
This commit is contained in:
Athou
2025-01-27 07:35:57 +01:00
parent de441e4ff7
commit 37cf711cbc
6 changed files with 107 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java
View File

@@ -1,5 +1,7 @@
package com.commafeed;
import java.time.InstantSource;
import com.codahale.metrics.MetricRegistry;
import jakarta.enterprise.inject.Produces;
@@ -8,9 +10,16 @@ import jakarta.inject.Singleton;
@Singleton
public class CommaFeedProducers {
@Produces
@Singleton
public InstantSource instantSource() {
return InstantSource.system();
}
@Produces
@Singleton
public MetricRegistry metricRegistry() {
return new MetricRegistry();
}
}

50
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -7,9 +7,12 @@ import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import org.apache.commons.lang3.StringUtils;
@@ -25,6 +28,7 @@ import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuil
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.client5.http.protocol.RedirectLocations;
import org.apache.hc.client5.http.utils.DateUtils;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpEntity;
@@ -51,6 +55,7 @@ import jakarta.ws.rs.core.CacheControl;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import nl.altindag.ssl.SSLFactory;
@@ -64,11 +69,13 @@ import nl.altindag.ssl.apache5.util.Apache5SslUtils;
public class HttpGetter {
private final CommaFeedConfiguration config;
private final InstantSource instantSource;
private final CloseableHttpClient client;
private final Cache<HttpRequest, HttpResponse> cache;
public HttpGetter(CommaFeedConfiguration config, CommaFeedVersion version, MetricRegistry metrics) {
public HttpGetter(CommaFeedConfiguration config, InstantSource instantSource, CommaFeedVersion version, MetricRegistry metrics) {
this.config = config;
this.instantSource = instantSource;
PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config);
String userAgent = config.httpClient()
@@ -88,11 +95,11 @@ public class HttpGetter {
() -> cache == null ? 0 : cache.asMap().values().stream().mapToInt(e -> e.content != null ? e.content.length : 0).sum());
}
public HttpResult get(String url) throws IOException, NotModifiedException {
public HttpResult get(String url) throws IOException, NotModifiedException, TooManyRequestsException {
return get(HttpRequest.builder(url).build());
}
public HttpResult get(HttpRequest request) throws IOException, NotModifiedException {
public HttpResult get(HttpRequest request) throws IOException, NotModifiedException, TooManyRequestsException {
final HttpResponse response;
if (cache == null) {
response = invoke(request);
@@ -109,9 +116,15 @@ public class HttpGetter {
}
int code = response.getCode();
if (Set.of(HttpStatus.SC_TOO_MANY_REQUESTS, HttpStatus.SC_SERVICE_UNAVAILABLE).contains(code) && response.getRetryAfter() != null) {
throw new TooManyRequestsException(response.getRetryAfter());
}
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
} else if (code >= 300) {
}
if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
@@ -165,6 +178,12 @@ public class HttpGetter {
.map(HttpGetter::toCacheControl)
.orElse(null);
Instant retryAfter = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(this::toInstant)
.orElse(null);
String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null);
String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations())
.map(RedirectLocations::getAll)
@@ -172,7 +191,7 @@ public class HttpGetter {
.map(URI::toString)
.orElse(request.getUrl());
return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, content, contentType, urlAfterRedirect);
return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, retryAfter, content, contentType, urlAfterRedirect);
});
}
@@ -185,6 +204,18 @@ public class HttpGetter {
}
}
private Instant toInstant(String headerValue) {
if (headerValue == null) {
return null;
}
if (StringUtils.isNumeric(headerValue)) {
return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
}
return DateUtils.parseStandardDate(headerValue);
}
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
@@ -292,6 +323,14 @@ public class HttpGetter {
}
}
@RequiredArgsConstructor
@Getter
public static class TooManyRequestsException extends Exception {
private static final long serialVersionUID = 1L;
private final Instant retryAfter;
}
@Getter
public static class HttpResponseException extends IOException {
private static final long serialVersionUID = 1L;
@@ -334,6 +373,7 @@ public class HttpGetter {
String lastModifiedHeader;
String eTagHeader;
CacheControl cacheControl;
Instant retryAfter;
byte[] content;
String contentType;
String urlAfterRedirect;

9
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -13,6 +13,7 @@ import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.model.Feed;
import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
@@ -91,7 +92,7 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
return new Favicon(bytes, contentType);
}
private byte[] fetchForUser(String googleAuthKey, String userId) throws IOException, NotModifiedException {
private byte[] fetchForUser(String googleAuthKey, String userId) throws IOException, NotModifiedException, TooManyRequestsException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
@@ -100,7 +101,8 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
return getter.get(uri.toString()).getContent();
}
private byte[] fetchForChannel(String googleAuthKey, String channelId) throws IOException, NotModifiedException {
private byte[] fetchForChannel(String googleAuthKey, String channelId)
throws IOException, NotModifiedException, TooManyRequestsException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
@@ -109,7 +111,8 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
return getter.get(uri.toString()).getContent();
}
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId) throws IOException, NotModifiedException {
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
throws IOException, NotModifiedException, TooManyRequestsException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)

4
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -13,6 +13,7 @@ import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpRequest;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
@@ -40,7 +41,8 @@ public class FeedFetcher {
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedException, IOException, NotModifiedException {
Instant lastPublishedDate, String lastContentHash)
throws FeedException, IOException, NotModifiedException, TooManyRequestsException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());

9
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -13,6 +13,7 @@ import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
@@ -97,6 +98,14 @@ public class FeedRefreshWorker {
feed.setEtagHeader(e.getNewEtagHeader());
}
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (TooManyRequestsException e) {
log.debug("Too many requests : {}", feed.getUrl());
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(e.getRetryAfter());
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
log.debug("unable to refresh feed {}", feed.getUrl(), e);