This commit is contained in:
Athou
2025-09-27 19:07:39 +02:00
parent 3b4cc66b24
commit c548462eef
7 changed files with 68 additions and 88 deletions

View File

@@ -58,7 +58,6 @@ import lombok.EqualsAndHashCode;
import lombok.Getter; import lombok.Getter;
import lombok.Lombok; import lombok.Lombok;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import nl.altindag.ssl.SSLFactory; import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils; import nl.altindag.ssl.apache5.util.Apache5SslUtils;
@@ -127,9 +126,9 @@ public class HttpGetter {
} }
} }
int code = response.getCode(); int code = response.code();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.getRetryAfter() != null) { if (code == HttpStatus.SC_TOO_MANY_REQUESTS || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.retryAfter() != null) {
throw new TooManyRequestsException(response.getRetryAfter()); throw new TooManyRequestsException(response.retryAfter());
} }
if (code == HttpStatus.SC_NOT_MODIFIED) { if (code == HttpStatus.SC_NOT_MODIFIED) {
@@ -140,16 +139,16 @@ public class HttpGetter {
throw new HttpResponseException(code, "Server returned HTTP error code " + code); throw new HttpResponseException(code, "Server returned HTTP error code " + code);
} }
String lastModifiedHeader = response.getLastModifiedHeader(); String lastModifiedHeader = response.lastModifiedHeader();
String eTagHeader = response.getETagHeader(); String eTagHeader = response.eTagHeader();
Duration validFor = Optional.ofNullable(response.getCacheControl()) Duration validFor = Optional.ofNullable(response.cacheControl())
.filter(cc -> cc.getMaxAge() >= 0) .filter(cc -> cc.getMaxAge() >= 0)
.map(cc -> Duration.ofSeconds(cc.getMaxAge())) .map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO); .orElse(Duration.ZERO);
return new HttpResult(response.getContent(), response.getContentType(), lastModifiedHeader, eTagHeader, return new HttpResult(response.content(), response.contentType(), lastModifiedHeader, eTagHeader, response.urlAfterRedirect(),
response.getUrlAfterRedirect(), validFor); validFor);
} }
private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException { private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
@@ -242,28 +241,25 @@ public class HttpGetter {
return DateUtils.parseStandardDate(headerValue); return DateUtils.parseStandardDate(headerValue);
} }
// ByteStreams.limit(input, maxBytes) reads at most maxBytes bytes.
// If the content length is exactly maxBytes, it throws an exception, even though the response is valid.
// This is an off-by-one error.
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)".formatted(entity.getContentLength(), maxBytes));
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes(); // read one extra to detect overflow
if (bytes.length > maxBytes) {
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes));
}
return bytes;
}
}
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)".formatted(entity.getContentLength(), maxBytes));
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes();
if (bytes.length > maxBytes) {
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes));
}
return bytes;
}
}
private PoolingHttpClientConnectionManager newConnectionManager(CommaFeedConfiguration config) { private PoolingHttpClientConnectionManager newConnectionManager(CommaFeedConfiguration config) {
SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build(); SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
@@ -310,7 +306,7 @@ public class HttpGetter {
} }
return CacheBuilder.newBuilder() return CacheBuilder.newBuilder()
.weigher((HttpRequest key, HttpResponse value) -> value.getContent() != null ? value.getContent().length : 0) .weigher((HttpRequest key, HttpResponse value) -> value.content() != null ? value.content().length : 0)
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue()) .maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
.expireAfterWrite(cacheConfig.expiration()) .expireAfterWrite(cacheConfig.expiration())
.build(); .build();
@@ -401,26 +397,12 @@ public class HttpGetter {
} }
} }
@Value private record HttpResponse(int code, String lastModifiedHeader, String eTagHeader, CacheControl cacheControl, Instant retryAfter,
private static class HttpResponse { byte[] content, String contentType, String urlAfterRedirect) {
int code;
String lastModifiedHeader;
String eTagHeader;
CacheControl cacheControl;
Instant retryAfter;
byte[] content;
String contentType;
String urlAfterRedirect;
} }
@Value public record HttpResult(byte[] content, String contentType, String lastModifiedSince, String eTag, String urlAfterRedirect,
public static class HttpResult { Duration validFor) {
byte[] content;
String contentType;
String lastModifiedSince;
String eTag;
String urlAfterRedirect;
Duration validFor;
} }
} }

View File

@@ -71,8 +71,8 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
url = Urls.removeTrailingSlash(url) + "/favicon.ico"; url = Urls.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url); log.debug("getting root icon at {}", url);
HttpResult result = getter.get(url); HttpResult result = getter.get(url);
bytes = result.getContent(); bytes = result.content();
contentType = result.getContentType(); contentType = result.contentType();
} catch (Exception e) { } catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url); log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e); log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
@@ -89,7 +89,7 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
Document doc; Document doc;
try { try {
HttpResult result = getter.get(url); HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.getContent()), url); doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) { } catch (Exception e) {
log.debug("Failed to retrieve page to find icon"); log.debug("Failed to retrieve page to find icon");
log.trace("Failed to retrieve page to find icon", e); log.trace("Failed to retrieve page to find icon", e);
@@ -115,8 +115,8 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
String contentType; String contentType;
try { try {
HttpResult result = getter.get(href); HttpResult result = getter.get(href);
bytes = result.getContent(); bytes = result.content();
contentType = result.getContentType(); contentType = result.contentType();
} catch (Exception e) { } catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href); log.debug("Failed to retrieve icon found in page {}", href);
log.trace("Failed to retrieve icon found in page {}", href, e); log.trace("Failed to retrieve icon found in page {}", href, e);

View File

@@ -45,8 +45,8 @@ public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
log.debug("Getting Facebook user's icon, {}", url); log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl); HttpResult iconResult = getter.get(iconUrl);
bytes = iconResult.getContent(); bytes = iconResult.content();
contentType = iconResult.getContentType(); contentType = iconResult.contentType();
} catch (Exception e) { } catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e); log.debug("Failed to retrieve Facebook icon", e);
} }

View File

@@ -85,8 +85,8 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
} }
HttpResult iconResult = getter.get(thumbnailUrl.asText()); HttpResult iconResult = getter.get(thumbnailUrl.asText());
bytes = iconResult.getContent(); bytes = iconResult.content();
contentType = iconResult.getContentType(); contentType = iconResult.contentType();
} catch (Exception e) { } catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e); log.debug("Failed to retrieve YouTube icon", e);
} }
@@ -104,7 +104,7 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
.queryParam("key", googleAuthKey) .queryParam("key", googleAuthKey)
.queryParam("forUsername", userId) .queryParam("forUsername", userId)
.build(); .build();
return getter.get(uri.toString()).getContent(); return getter.get(uri.toString()).content();
} }
private byte[] fetchForChannel(String googleAuthKey, String channelId) private byte[] fetchForChannel(String googleAuthKey, String channelId)
@@ -114,7 +114,7 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
.queryParam("key", googleAuthKey) .queryParam("key", googleAuthKey)
.queryParam("id", channelId) .queryParam("id", channelId)
.build(); .build();
return getter.get(uri.toString()).getContent(); return getter.get(uri.toString()).content();
} }
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId) private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
@@ -124,7 +124,7 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
.queryParam("key", googleAuthKey) .queryParam("key", googleAuthKey)
.queryParam("id", playlistId) .queryParam("id", playlistId)
.build(); .build();
byte[] playlistBytes = getter.get(uri.toString()).getContent(); byte[] playlistBytes = getter.get(uri.toString()).content();
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID); JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
if (channelId.isMissingNode()) { if (channelId.isMissingNode()) {

View File

@@ -50,20 +50,20 @@ public class FeedFetcher {
log.debug("Fetching feed {}", feedUrl); log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build()); HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.getContent(); byte[] content = result.content();
FeedParserResult parserResult; FeedParserResult parserResult;
try { try {
parserResult = parser.parse(result.getUrlAfterRedirect(), content); parserResult = parser.parse(result.urlAfterRedirect(), content);
} catch (FeedParsingException e) { } catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) { if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.getContent(), StandardCharsets.UTF_8)); String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.content(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) { if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl; feedUrl = extractedUrl;
result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build()); result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build());
content = result.getContent(); content = result.content();
parserResult = parser.parse(result.getUrlAfterRedirect(), content); parserResult = parser.parse(result.urlAfterRedirect(), content);
} else { } else {
throw new NoFeedFoundException(e); throw new NoFeedFoundException(e);
} }
@@ -76,26 +76,24 @@ public class FeedFetcher {
throw new IOException("Feed content is empty."); throw new IOException("Feed content is empty.");
} }
boolean lastModifiedHeaderValueChanged = !Strings.CS.equals(lastModified, result.getLastModifiedSince()); boolean lastModifiedHeaderValueChanged = !Strings.CS.equals(lastModified, result.lastModifiedSince());
boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.getETag()); boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.eTag());
String hash = Digests.sha1Hex(content); String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) { if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl); log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified", throw new NotModifiedException("content hash not modified", lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null, etagHeaderValueChanged ? result.eTag() : null);
etagHeaderValueChanged ? result.getETag() : null);
} }
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) { if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl); log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified", throw new NotModifiedException("publishedDate not modified", lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null, etagHeaderValueChanged ? result.eTag() : null);
etagHeaderValueChanged ? result.getETag() : null);
} }
return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash, return new FeedFetcherResult(parserResult, result.urlAfterRedirect(), result.lastModifiedSince(), result.eTag(), hash,
result.getValidFor()); result.validFor());
} }
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) { private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {

View File

@@ -74,7 +74,7 @@ public class ServerREST {
url = ImageProxyUrl.decode(url); url = ImageProxyUrl.decode(url);
try { try {
HttpResult result = httpGetter.get(url); HttpResult result = httpGetter.get(url);
return Response.ok(result.getContent()).build(); return Response.ok(result.content()).build();
} catch (Exception e) { } catch (Exception e) {
return Response.status(Status.SERVICE_UNAVAILABLE).entity(e.getMessage()).build(); return Response.status(Status.SERVICE_UNAVAILABLE).entity(e.getMessage()).build();
} }

View File

@@ -104,12 +104,12 @@ class HttpGetterTest {
.withHeader(HttpHeaders.RETRY_AFTER, "120")); .withHeader(HttpHeaders.RETRY_AFTER, "120"));
HttpResult result = getter.get(this.feedUrl); HttpResult result = getter.get(this.feedUrl);
Assertions.assertArrayEquals(feedContent, result.getContent()); Assertions.assertArrayEquals(feedContent, result.content());
Assertions.assertEquals(MediaType.APPLICATION_ATOM_XML.toString(), result.getContentType()); Assertions.assertEquals(MediaType.APPLICATION_ATOM_XML.toString(), result.contentType());
Assertions.assertEquals("123456", result.getLastModifiedSince()); Assertions.assertEquals("123456", result.lastModifiedSince());
Assertions.assertEquals("78910", result.getETag()); Assertions.assertEquals("78910", result.eTag());
Assertions.assertEquals(Duration.ofSeconds(60), result.getValidFor()); Assertions.assertEquals(Duration.ofSeconds(60), result.validFor());
Assertions.assertEquals(this.feedUrl, result.getUrlAfterRedirect()); Assertions.assertEquals(this.feedUrl, result.urlAfterRedirect());
} }
@Test @Test
@@ -121,7 +121,7 @@ class HttpGetterTest {
.withHeader(HttpHeaders.CACHE_CONTROL, "max-age=60; must-revalidate")); .withHeader(HttpHeaders.CACHE_CONTROL, "max-age=60; must-revalidate"));
HttpResult result = getter.get(this.feedUrl); HttpResult result = getter.get(this.feedUrl);
Assertions.assertEquals(Duration.ZERO, result.getValidFor()); Assertions.assertEquals(Duration.ZERO, result.validFor());
} }
@Test @Test
@@ -167,7 +167,7 @@ class HttpGetterTest {
.respond(HttpResponse.response().withBody(feedContent).withContentType(MediaType.APPLICATION_ATOM_XML)); .respond(HttpResponse.response().withBody(feedContent).withContentType(MediaType.APPLICATION_ATOM_XML));
HttpResult result = getter.get(this.feedUrl); HttpResult result = getter.get(this.feedUrl);
Assertions.assertEquals("http://localhost:" + this.mockServerClient.getPort() + "/redirected-2", result.getUrlAfterRedirect()); Assertions.assertEquals("http://localhost:" + this.mockServerClient.getPort() + "/redirected-2", result.urlAfterRedirect());
} }
@Test @Test
@@ -202,7 +202,7 @@ class HttpGetterTest {
.respond(HttpResponse.response().withBody("ok")); .respond(HttpResponse.response().withBody("ok"));
HttpResult result = getter.get(this.feedUrl); HttpResult result = getter.get(this.feedUrl);
Assertions.assertEquals("ok", new String(result.getContent())); Assertions.assertEquals("ok", new String(result.content()));
} }
@Test @Test
@@ -284,7 +284,7 @@ class HttpGetterTest {
this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(HttpResponse.response().withBody("ok")); this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(HttpResponse.response().withBody("ok"));
HttpResult result = getter.get("https://localhost:" + this.mockServerClient.getPort()); HttpResult result = getter.get("https://localhost:" + this.mockServerClient.getPort());
Assertions.assertEquals("ok", new String(result.getContent())); Assertions.assertEquals("ok", new String(result.content()));
} }
@Test @Test
@@ -336,7 +336,7 @@ class HttpGetterTest {
}); });
HttpResult result = getter.get(HttpGetterTest.this.feedUrl); HttpResult result = getter.get(HttpGetterTest.this.feedUrl);
Assertions.assertEquals(body, new String(result.getContent())); Assertions.assertEquals(body, new String(result.content()));
} }
@FunctionalInterface @FunctionalInterface