garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

cleanup

Browse Source
This commit is contained in:
Athou
2025-09-27 19:07:39 +02:00
parent 3b4cc66b24
commit c548462eef
7 changed files with 68 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -58,7 +58,6 @@ import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Lombok;
import lombok.RequiredArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
@@ -127,9 +126,9 @@ public class HttpGetter {
}
}
int code = response.getCode();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.getRetryAfter() != null) {
throw new TooManyRequestsException(response.getRetryAfter());
int code = response.code();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.retryAfter() != null) {
throw new TooManyRequestsException(response.retryAfter());
}
if (code == HttpStatus.SC_NOT_MODIFIED) {
@@ -140,16 +139,16 @@ public class HttpGetter {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
String lastModifiedHeader = response.getLastModifiedHeader();
String eTagHeader = response.getETagHeader();
String lastModifiedHeader = response.lastModifiedHeader();
String eTagHeader = response.eTagHeader();
Duration validFor = Optional.ofNullable(response.getCacheControl())
Duration validFor = Optional.ofNullable(response.cacheControl())
.filter(cc -> cc.getMaxAge() >= 0)
.map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO);
return new HttpResult(response.getContent(), response.getContentType(), lastModifiedHeader, eTagHeader,
response.getUrlAfterRedirect(), validFor);
return new HttpResult(response.content(), response.contentType(), lastModifiedHeader, eTagHeader, response.urlAfterRedirect(),
validFor);
}
private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
@@ -242,28 +241,25 @@ public class HttpGetter {
return DateUtils.parseStandardDate(headerValue);
}
// ByteStreams.limit(input, maxBytes) reads at most maxBytes bytes.
// If the content length is exactly maxBytes, it throws an exception, even though the response is valid.
// This is an off-by-one error.
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)".formatted(entity.getContentLength(), maxBytes));
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes(); // read one extra to detect overflow
if (bytes.length > maxBytes) {
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes));
}
return bytes;
}
}
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)".formatted(entity.getContentLength(), maxBytes));
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes();
if (bytes.length > maxBytes) {
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes));
}
return bytes;
}
}
private PoolingHttpClientConnectionManager newConnectionManager(CommaFeedConfiguration config) {
SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
@@ -310,7 +306,7 @@ public class HttpGetter {
}
return CacheBuilder.newBuilder()
.weigher((HttpRequest key, HttpResponse value) -> value.getContent() != null ? value.getContent().length : 0)
.weigher((HttpRequest key, HttpResponse value) -> value.content() != null ? value.content().length : 0)
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
.expireAfterWrite(cacheConfig.expiration())
.build();
@@ -401,26 +397,12 @@ public class HttpGetter {
}
}
@Value
private static class HttpResponse {
int code;
String lastModifiedHeader;
String eTagHeader;
CacheControl cacheControl;
Instant retryAfter;
byte[] content;
String contentType;
String urlAfterRedirect;
private record HttpResponse(int code, String lastModifiedHeader, String eTagHeader, CacheControl cacheControl, Instant retryAfter,
byte[] content, String contentType, String urlAfterRedirect) {
}
@Value
public static class HttpResult {
byte[] content;
String contentType;
String lastModifiedSince;
String eTag;
String urlAfterRedirect;
Duration validFor;
public record HttpResult(byte[] content, String contentType, String lastModifiedSince, String eTag, String urlAfterRedirect,
Duration validFor) {
}
}

10
commafeed-server/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
View File

@@ -71,8 +71,8 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
url = Urls.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url);
HttpResult result = getter.get(url);
bytes = result.getContent();
contentType = result.getContentType();
bytes = result.content();
contentType = result.contentType();
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
@@ -89,7 +89,7 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.getContent()), url);
doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon");
log.trace("Failed to retrieve page to find icon", e);
@@ -115,8 +115,8 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
String contentType;
try {
HttpResult result = getter.get(href);
bytes = result.getContent();
contentType = result.getContentType();
bytes = result.content();
contentType = result.contentType();
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href);
log.trace("Failed to retrieve icon found in page {}", href, e);

4
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
View File

@@ -45,8 +45,8 @@ public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl);
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
bytes = iconResult.content();
contentType = iconResult.contentType();
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
}

10
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -85,8 +85,8 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
bytes = iconResult.content();
contentType = iconResult.contentType();
} catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e);
}
@@ -104,7 +104,7 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
.queryParam("key", googleAuthKey)
.queryParam("forUsername", userId)
.build();
return getter.get(uri.toString()).getContent();
return getter.get(uri.toString()).content();
}
private byte[] fetchForChannel(String googleAuthKey, String channelId)
@@ -114,7 +114,7 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
.queryParam("key", googleAuthKey)
.queryParam("id", channelId)
.build();
return getter.get(uri.toString()).getContent();
return getter.get(uri.toString()).content();
}
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
@@ -124,7 +124,7 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
.queryParam("key", googleAuthKey)
.queryParam("id", playlistId)
.build();
byte[] playlistBytes = getter.get(uri.toString()).getContent();
byte[] playlistBytes = getter.get(uri.toString()).content();
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
if (channelId.isMissingNode()) {

28
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -50,20 +50,20 @@ public class FeedFetcher {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.getContent();
byte[] content = result.content();
FeedParserResult parserResult;
try {
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
parserResult = parser.parse(result.urlAfterRedirect(), content);
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.getContent(), StandardCharsets.UTF_8));
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.content(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build());
content = result.getContent();
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
content = result.content();
parserResult = parser.parse(result.urlAfterRedirect(), content);
} else {
throw new NoFeedFoundException(e);
}
@@ -76,26 +76,24 @@ public class FeedFetcher {
throw new IOException("Feed content is empty.");
}
boolean lastModifiedHeaderValueChanged = !Strings.CS.equals(lastModified, result.getLastModifiedSince());
boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.getETag());
boolean lastModifiedHeaderValueChanged = !Strings.CS.equals(lastModified, result.lastModifiedSince());
boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.eTag());
String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
throw new NotModifiedException("content hash not modified", lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
throw new NotModifiedException("publishedDate not modified", lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash,
result.getValidFor());
return new FeedFetcherResult(parserResult, result.urlAfterRedirect(), result.lastModifiedSince(), result.eTag(), hash,
result.validFor());
}
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {

2
commafeed-server/src/main/java/com/commafeed/frontend/resource/ServerREST.java
View File

@@ -74,7 +74,7 @@ public class ServerREST {
url = ImageProxyUrl.decode(url);
try {
HttpResult result = httpGetter.get(url);
return Response.ok(result.getContent()).build();
return Response.ok(result.content()).build();
} catch (Exception e) {
return Response.status(Status.SERVICE_UNAVAILABLE).entity(e.getMessage()).build();
}

22
commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java
View File

@@ -104,12 +104,12 @@ class HttpGetterTest {
.withHeader(HttpHeaders.RETRY_AFTER, "120"));
HttpResult result = getter.get(this.feedUrl);
Assertions.assertArrayEquals(feedContent, result.getContent());
Assertions.assertEquals(MediaType.APPLICATION_ATOM_XML.toString(), result.getContentType());
Assertions.assertEquals("123456", result.getLastModifiedSince());
Assertions.assertEquals("78910", result.getETag());
Assertions.assertEquals(Duration.ofSeconds(60), result.getValidFor());
Assertions.assertEquals(this.feedUrl, result.getUrlAfterRedirect());
Assertions.assertArrayEquals(feedContent, result.content());
Assertions.assertEquals(MediaType.APPLICATION_ATOM_XML.toString(), result.contentType());
Assertions.assertEquals("123456", result.lastModifiedSince());
Assertions.assertEquals("78910", result.eTag());
Assertions.assertEquals(Duration.ofSeconds(60), result.validFor());
Assertions.assertEquals(this.feedUrl, result.urlAfterRedirect());
}
@Test
@@ -121,7 +121,7 @@ class HttpGetterTest {
.withHeader(HttpHeaders.CACHE_CONTROL, "max-age=60; must-revalidate"));
HttpResult result = getter.get(this.feedUrl);
Assertions.assertEquals(Duration.ZERO, result.getValidFor());
Assertions.assertEquals(Duration.ZERO, result.validFor());
}
@Test
@@ -167,7 +167,7 @@ class HttpGetterTest {
.respond(HttpResponse.response().withBody(feedContent).withContentType(MediaType.APPLICATION_ATOM_XML));
HttpResult result = getter.get(this.feedUrl);
Assertions.assertEquals("http://localhost:" + this.mockServerClient.getPort() + "/redirected-2", result.getUrlAfterRedirect());
Assertions.assertEquals("http://localhost:" + this.mockServerClient.getPort() + "/redirected-2", result.urlAfterRedirect());
}
@Test
@@ -202,7 +202,7 @@ class HttpGetterTest {
.respond(HttpResponse.response().withBody("ok"));
HttpResult result = getter.get(this.feedUrl);
Assertions.assertEquals("ok", new String(result.getContent()));
Assertions.assertEquals("ok", new String(result.content()));
}
@Test
@@ -284,7 +284,7 @@ class HttpGetterTest {
this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(HttpResponse.response().withBody("ok"));
HttpResult result = getter.get("https://localhost:" + this.mockServerClient.getPort());
Assertions.assertEquals("ok", new String(result.getContent()));
Assertions.assertEquals("ok", new String(result.content()));
}
@Test
@@ -336,7 +336,7 @@ class HttpGetterTest {
});
HttpResult result = getter.get(HttpGetterTest.this.feedUrl);
Assertions.assertEquals(body, new String(result.getContent()));
Assertions.assertEquals(body, new String(result.content()));
}
@FunctionalInterface