mirror of
https://github.com/Athou/commafeed.git
synced 2026-11-03 10:54:46 +00:00
add support for HTTP/2 (#2317)
This commit is contained in:
@@ -456,11 +456,6 @@
|
|||||||
<artifactId>ayza-for-apache5</artifactId>
|
<artifactId>ayza-for-apache5</artifactId>
|
||||||
<version>10.1.0</version>
|
<version>10.1.0</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
|
||||||
<groupId>org.brotli</groupId>
|
|
||||||
<artifactId>dec</artifactId>
|
|
||||||
<version>0.1.2</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.github.seancfoley</groupId>
|
<groupId>com.github.seancfoley</groupId>
|
||||||
<artifactId>ipaddress</artifactId>
|
<artifactId>ipaddress</artifactId>
|
||||||
|
|||||||
@@ -19,27 +19,27 @@ import nl.altindag.ssl.apache5.util.Apache5SslUtils;
|
|||||||
|
|
||||||
import org.apache.hc.client5.http.DnsResolver;
|
import org.apache.hc.client5.http.DnsResolver;
|
||||||
import org.apache.hc.client5.http.SystemDefaultDnsResolver;
|
import org.apache.hc.client5.http.SystemDefaultDnsResolver;
|
||||||
|
import org.apache.hc.client5.http.async.methods.InflatingAsyncDataConsumer;
|
||||||
|
import org.apache.hc.client5.http.async.methods.InflatingGzipDataConsumer;
|
||||||
import org.apache.hc.client5.http.config.ConnectionConfig;
|
import org.apache.hc.client5.http.config.ConnectionConfig;
|
||||||
import org.apache.hc.client5.http.config.TlsConfig;
|
import org.apache.hc.client5.http.config.TlsConfig;
|
||||||
import org.apache.hc.client5.http.entity.DeflateInputStream;
|
|
||||||
import org.apache.hc.client5.http.entity.InputStreamFactory;
|
|
||||||
import org.apache.hc.client5.http.entity.compress.ContentCoding;
|
import org.apache.hc.client5.http.entity.compress.ContentCoding;
|
||||||
import org.apache.hc.client5.http.impl.DefaultRedirectStrategy;
|
import org.apache.hc.client5.http.impl.DefaultRedirectStrategy;
|
||||||
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
|
import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
|
||||||
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
|
import org.apache.hc.client5.http.impl.async.HttpAsyncClients;
|
||||||
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
|
import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager;
|
||||||
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
|
import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
|
||||||
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
|
|
||||||
import org.apache.hc.client5.http.protocol.RedirectStrategy;
|
import org.apache.hc.client5.http.protocol.RedirectStrategy;
|
||||||
import org.apache.hc.core5.http.Header;
|
import org.apache.hc.core5.http.Header;
|
||||||
import org.apache.hc.core5.http.HttpException;
|
import org.apache.hc.core5.http.HttpException;
|
||||||
import org.apache.hc.core5.http.HttpRequest;
|
import org.apache.hc.core5.http.HttpRequest;
|
||||||
import org.apache.hc.core5.http.HttpResponse;
|
import org.apache.hc.core5.http.HttpResponse;
|
||||||
import org.apache.hc.core5.http.message.BasicHeader;
|
import org.apache.hc.core5.http.message.BasicHeader;
|
||||||
|
import org.apache.hc.core5.http.nio.AsyncDataConsumer;
|
||||||
import org.apache.hc.core5.http.protocol.HttpContext;
|
import org.apache.hc.core5.http.protocol.HttpContext;
|
||||||
|
import org.apache.hc.core5.http2.HttpVersionPolicy;
|
||||||
import org.apache.hc.core5.util.TimeValue;
|
import org.apache.hc.core5.util.TimeValue;
|
||||||
import org.apache.hc.core5.util.Timeout;
|
import org.apache.hc.core5.util.Timeout;
|
||||||
import org.brotli.dec.BrotliInputStream;
|
|
||||||
|
|
||||||
import java.net.InetAddress;
|
import java.net.InetAddress;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
@@ -48,7 +48,7 @@ import java.util.ArrayList;
|
|||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.SequencedMap;
|
import java.util.SequencedMap;
|
||||||
import java.util.zip.GZIPInputStream;
|
import java.util.function.UnaryOperator;
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -60,8 +60,8 @@ public class HttpClientFactory {
|
|||||||
private final CommaFeedConfiguration config;
|
private final CommaFeedConfiguration config;
|
||||||
private final CommaFeedVersion version;
|
private final CommaFeedVersion version;
|
||||||
|
|
||||||
public CloseableHttpClient newClient(int poolSize) {
|
public CloseableHttpAsyncClient newClient(int poolSize) {
|
||||||
PoolingHttpClientConnectionManager connectionManager =
|
PoolingAsyncClientConnectionManager connectionManager =
|
||||||
newConnectionManager(config, poolSize);
|
newConnectionManager(config, poolSize);
|
||||||
String userAgent =
|
String userAgent =
|
||||||
config.httpClient()
|
config.httpClient()
|
||||||
@@ -74,42 +74,46 @@ public class HttpClientFactory {
|
|||||||
return newClient(config, connectionManager, userAgent);
|
return newClient(config, connectionManager, userAgent);
|
||||||
}
|
}
|
||||||
|
|
||||||
private CloseableHttpClient newClient(
|
private CloseableHttpAsyncClient newClient(
|
||||||
CommaFeedConfiguration config,
|
CommaFeedConfiguration config,
|
||||||
HttpClientConnectionManager connectionManager,
|
PoolingAsyncClientConnectionManager connectionManager,
|
||||||
String userAgent) {
|
String userAgent) {
|
||||||
List<Header> headers = new ArrayList<>();
|
List<Header> headers = new ArrayList<>();
|
||||||
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
|
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
|
||||||
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
|
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
|
||||||
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
|
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
|
||||||
|
|
||||||
SequencedMap<String, InputStreamFactory> contentDecoderMap = new LinkedHashMap<>();
|
SequencedMap<String, UnaryOperator<AsyncDataConsumer>> contentDecoderMap =
|
||||||
contentDecoderMap.put(ContentCoding.GZIP.token(), GZIPInputStream::new);
|
new LinkedHashMap<>();
|
||||||
contentDecoderMap.put(ContentCoding.DEFLATE.token(), DeflateInputStream::new);
|
contentDecoderMap.put(
|
||||||
contentDecoderMap.put(ContentCoding.BROTLI.token(), BrotliInputStream::new);
|
ContentCoding.DEFLATE.token(), d -> new InflatingAsyncDataConsumer(d, null));
|
||||||
|
contentDecoderMap.put(ContentCoding.GZIP.token(), InflatingGzipDataConsumer::new);
|
||||||
|
|
||||||
RedirectStrategy redirectStrategy =
|
RedirectStrategy redirectStrategy =
|
||||||
config.httpClient().blockLocalAddresses()
|
config.httpClient().blockLocalAddresses()
|
||||||
? new BlockLocalAddressesRedirectStrategy(DNS_RESOLVER)
|
? new BlockLocalAddressesRedirectStrategy(DNS_RESOLVER)
|
||||||
: new DefaultRedirectStrategy();
|
: new DefaultRedirectStrategy();
|
||||||
|
|
||||||
return HttpClientBuilder.create()
|
CloseableHttpAsyncClient client =
|
||||||
.disableConnectionState()
|
HttpAsyncClients.custom()
|
||||||
.useSystemProperties()
|
.disableConnectionState()
|
||||||
.disableAutomaticRetries()
|
.useSystemProperties()
|
||||||
.disableCookieManagement()
|
.disableAutomaticRetries()
|
||||||
.setUserAgent(userAgent)
|
.disableCookieManagement()
|
||||||
.setDefaultHeaders(headers)
|
.setUserAgent(userAgent)
|
||||||
.setConnectionManager(connectionManager)
|
.setDefaultHeaders(headers)
|
||||||
.evictExpiredConnections()
|
.setConnectionManager(connectionManager)
|
||||||
.evictIdleConnections(
|
.evictExpiredConnections()
|
||||||
TimeValue.of(config.httpClient().idleConnectionsEvictionInterval()))
|
.evictIdleConnections(
|
||||||
.setContentDecoderRegistry(new LinkedHashMap<>(contentDecoderMap))
|
TimeValue.of(config.httpClient().idleConnectionsEvictionInterval()))
|
||||||
.setRedirectStrategy(redirectStrategy)
|
.setContentDecoderMap(new LinkedHashMap<>(contentDecoderMap))
|
||||||
.build();
|
.setRedirectStrategy(redirectStrategy)
|
||||||
|
.build();
|
||||||
|
client.start();
|
||||||
|
return client;
|
||||||
}
|
}
|
||||||
|
|
||||||
private PoolingHttpClientConnectionManager newConnectionManager(
|
private PoolingAsyncClientConnectionManager newConnectionManager(
|
||||||
CommaFeedConfiguration config, int poolSize) {
|
CommaFeedConfiguration config, int poolSize) {
|
||||||
SSLFactory sslFactory =
|
SSLFactory sslFactory =
|
||||||
SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
|
SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
|
||||||
@@ -118,8 +122,8 @@ public class HttpClientFactory {
|
|||||||
? new BlockLocalAddressesDnsResolver(DNS_RESOLVER)
|
? new BlockLocalAddressesDnsResolver(DNS_RESOLVER)
|
||||||
: DNS_RESOLVER;
|
: DNS_RESOLVER;
|
||||||
|
|
||||||
return PoolingHttpClientConnectionManagerBuilder.create()
|
return PoolingAsyncClientConnectionManagerBuilder.create()
|
||||||
.setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory))
|
.setTlsStrategy(Apache5SslUtils.toTlsStrategy(sslFactory))
|
||||||
.setDefaultConnectionConfig(
|
.setDefaultConnectionConfig(
|
||||||
ConnectionConfig.custom()
|
ConnectionConfig.custom()
|
||||||
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
|
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
|
||||||
@@ -129,6 +133,7 @@ public class HttpClientFactory {
|
|||||||
.build())
|
.build())
|
||||||
.setDefaultTlsConfig(
|
.setDefaultTlsConfig(
|
||||||
TlsConfig.custom()
|
TlsConfig.custom()
|
||||||
|
.setVersionPolicy(HttpVersionPolicy.NEGOTIATE)
|
||||||
.setHandshakeTimeout(
|
.setHandshakeTimeout(
|
||||||
Timeout.of(config.httpClient().sslHandshakeTimeout()))
|
Timeout.of(config.httpClient().sslHandshakeTimeout()))
|
||||||
.build())
|
.build())
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import com.commafeed.CommaFeedConfiguration.HttpClientCache;
|
|||||||
import com.google.common.cache.Cache;
|
import com.google.common.cache.Cache;
|
||||||
import com.google.common.cache.CacheBuilder;
|
import com.google.common.cache.CacheBuilder;
|
||||||
import com.google.common.collect.Iterables;
|
import com.google.common.collect.Iterables;
|
||||||
import com.google.common.io.ByteStreams;
|
|
||||||
import com.google.common.net.HttpHeaders;
|
import com.google.common.net.HttpHeaders;
|
||||||
|
|
||||||
import jakarta.inject.Singleton;
|
import jakarta.inject.Singleton;
|
||||||
@@ -21,35 +20,44 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
|
|
||||||
import org.apache.commons.lang3.ArrayUtils;
|
import org.apache.commons.lang3.ArrayUtils;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.hc.client5.http.async.methods.SimpleHttpRequest;
|
||||||
|
import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder;
|
||||||
|
import org.apache.hc.client5.http.async.methods.SimpleRequestProducer;
|
||||||
import org.apache.hc.client5.http.config.RequestConfig;
|
import org.apache.hc.client5.http.config.RequestConfig;
|
||||||
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
|
import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
|
||||||
import org.apache.hc.client5.http.protocol.HttpClientContext;
|
import org.apache.hc.client5.http.protocol.HttpClientContext;
|
||||||
import org.apache.hc.client5.http.protocol.RedirectLocations;
|
import org.apache.hc.client5.http.protocol.RedirectLocations;
|
||||||
import org.apache.hc.client5.http.utils.DateUtils;
|
import org.apache.hc.client5.http.utils.DateUtils;
|
||||||
import org.apache.hc.core5.http.ClassicHttpRequest;
|
import org.apache.hc.core5.concurrent.FutureCallback;
|
||||||
import org.apache.hc.core5.http.HttpEntity;
|
import org.apache.hc.core5.http.ContentType;
|
||||||
import org.apache.hc.core5.http.HttpStatus;
|
import org.apache.hc.core5.http.HttpStatus;
|
||||||
|
import org.apache.hc.core5.http.Message;
|
||||||
import org.apache.hc.core5.http.NameValuePair;
|
import org.apache.hc.core5.http.NameValuePair;
|
||||||
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
|
import org.apache.hc.core5.http.nio.AsyncRequestProducer;
|
||||||
|
import org.apache.hc.core5.http.nio.AsyncResponseConsumer;
|
||||||
|
import org.apache.hc.core5.http.nio.entity.AbstractBinAsyncEntityConsumer;
|
||||||
|
import org.apache.hc.core5.http.nio.support.BasicResponseConsumer;
|
||||||
import org.apache.hc.core5.util.Timeout;
|
import org.apache.hc.core5.util.Timeout;
|
||||||
import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate;
|
import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate;
|
||||||
|
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.time.InstantSource;
|
import java.time.InstantSource;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.concurrent.ExecutionException;
|
||||||
|
import java.util.concurrent.Future;
|
||||||
|
|
||||||
/** Smart HTTP getter: handles gzip, ssl, last modified and etag headers */
|
/** Smart HTTP getter: handles gzip, ssl, last modified, and etag headers */
|
||||||
@Singleton
|
@Singleton
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class HttpGetter {
|
public class HttpGetter {
|
||||||
private final CommaFeedConfiguration config;
|
private final CommaFeedConfiguration config;
|
||||||
private final InstantSource instantSource;
|
private final InstantSource instantSource;
|
||||||
private final CloseableHttpClient client;
|
private final CloseableHttpAsyncClient client;
|
||||||
private final Cache<HttpRequest, HttpResponse> cache;
|
private final Cache<HttpRequest, HttpResponse> cache;
|
||||||
|
|
||||||
public HttpGetter(
|
public HttpGetter(
|
||||||
@@ -157,63 +165,83 @@ public class HttpGetter {
|
|||||||
.setProtocolUpgradeEnabled(false)
|
.setProtocolUpgradeEnabled(false)
|
||||||
.build());
|
.build());
|
||||||
|
|
||||||
return client.execute(
|
AsyncRequestProducer req = SimpleRequestProducer.create(request.toSimpleHttpRequest());
|
||||||
request.toClassicHttpRequest(),
|
AsyncResponseConsumer<Message<org.apache.hc.core5.http.HttpResponse, byte[]>> resp =
|
||||||
context,
|
new BasicResponseConsumer<>(
|
||||||
resp -> {
|
new LimitedBytesEntityConsumer(
|
||||||
byte[] content =
|
config.httpClient().maxResponseSize().asLongValue()));
|
||||||
resp.getEntity() == null
|
FutureCallback<Message<org.apache.hc.core5.http.HttpResponse, byte[]>> callback =
|
||||||
? null
|
new FutureCallback<>() {
|
||||||
: toByteArray(
|
@Override
|
||||||
resp.getEntity(),
|
public void completed(
|
||||||
config.httpClient().maxResponseSize().asLongValue());
|
Message<org.apache.hc.core5.http.HttpResponse, byte[]> result) {}
|
||||||
int code = resp.getCode();
|
|
||||||
String lastModifiedHeader =
|
|
||||||
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
|
|
||||||
.map(NameValuePair::getValue)
|
|
||||||
.map(StringUtils::trimToNull)
|
|
||||||
.orElse(null);
|
|
||||||
String eTagHeader =
|
|
||||||
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
|
|
||||||
.map(NameValuePair::getValue)
|
|
||||||
.map(StringUtils::trimToNull)
|
|
||||||
.orElse(null);
|
|
||||||
|
|
||||||
CacheControl cacheControl =
|
@Override
|
||||||
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
|
public void failed(Exception ex) {}
|
||||||
.map(NameValuePair::getValue)
|
|
||||||
.map(StringUtils::trimToNull)
|
|
||||||
.map(HttpGetter::toCacheControl)
|
|
||||||
.orElse(null);
|
|
||||||
|
|
||||||
Instant retryAfter =
|
@Override
|
||||||
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
|
public void cancelled() {}
|
||||||
.map(NameValuePair::getValue)
|
};
|
||||||
.map(StringUtils::trimToNull)
|
Future<Message<org.apache.hc.core5.http.HttpResponse, byte[]>> future =
|
||||||
.map(this::toInstant)
|
client.execute(req, resp, context, callback);
|
||||||
.orElse(null);
|
|
||||||
|
|
||||||
String contentType =
|
try {
|
||||||
Optional.ofNullable(resp.getEntity())
|
Message<org.apache.hc.core5.http.HttpResponse, byte[]> message = future.get();
|
||||||
.map(HttpEntity::getContentType)
|
|
||||||
.orElse(null);
|
|
||||||
String urlAfterRedirect =
|
|
||||||
Optional.ofNullable(context.getRedirectLocations())
|
|
||||||
.map(RedirectLocations::getAll)
|
|
||||||
.map(l -> Iterables.getLast(l, null))
|
|
||||||
.map(URI::toString)
|
|
||||||
.orElse(request.getUrl());
|
|
||||||
|
|
||||||
return new HttpResponse(
|
byte[] content = message.getBody();
|
||||||
code,
|
int code = message.getHead().getCode();
|
||||||
lastModifiedHeader,
|
String lastModifiedHeader =
|
||||||
eTagHeader,
|
Optional.ofNullable(message.getHead().getFirstHeader(HttpHeaders.LAST_MODIFIED))
|
||||||
cacheControl,
|
.map(NameValuePair::getValue)
|
||||||
retryAfter,
|
.map(StringUtils::trimToNull)
|
||||||
content,
|
.orElse(null);
|
||||||
contentType,
|
String eTagHeader =
|
||||||
urlAfterRedirect);
|
Optional.ofNullable(message.getHead().getFirstHeader(HttpHeaders.ETAG))
|
||||||
});
|
.map(NameValuePair::getValue)
|
||||||
|
.map(StringUtils::trimToNull)
|
||||||
|
.orElse(null);
|
||||||
|
|
||||||
|
CacheControl cacheControl =
|
||||||
|
Optional.ofNullable(message.getHead().getFirstHeader(HttpHeaders.CACHE_CONTROL))
|
||||||
|
.map(NameValuePair::getValue)
|
||||||
|
.map(StringUtils::trimToNull)
|
||||||
|
.map(HttpGetter::toCacheControl)
|
||||||
|
.orElse(null);
|
||||||
|
|
||||||
|
Instant retryAfter =
|
||||||
|
Optional.ofNullable(message.getHead().getFirstHeader(HttpHeaders.RETRY_AFTER))
|
||||||
|
.map(NameValuePair::getValue)
|
||||||
|
.map(StringUtils::trimToNull)
|
||||||
|
.map(this::toInstant)
|
||||||
|
.orElse(null);
|
||||||
|
|
||||||
|
String contentType =
|
||||||
|
Optional.ofNullable(message.getHead().getFirstHeader(HttpHeaders.CONTENT_TYPE))
|
||||||
|
.map(NameValuePair::getValue)
|
||||||
|
.map(StringUtils::trimToNull)
|
||||||
|
.orElse(null);
|
||||||
|
String urlAfterRedirect =
|
||||||
|
Optional.ofNullable(context.getRedirectLocations())
|
||||||
|
.map(RedirectLocations::getAll)
|
||||||
|
.map(l -> Iterables.getLast(l, null))
|
||||||
|
.map(URI::toString)
|
||||||
|
.orElse(request.getUrl());
|
||||||
|
|
||||||
|
return new HttpResponse(
|
||||||
|
code,
|
||||||
|
lastModifiedHeader,
|
||||||
|
eTagHeader,
|
||||||
|
cacheControl,
|
||||||
|
retryAfter,
|
||||||
|
content,
|
||||||
|
contentType,
|
||||||
|
urlAfterRedirect);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
throw new RuntimeException("interrupted while fetching " + request.getUrl(), e);
|
||||||
|
} catch (ExecutionException e) {
|
||||||
|
throw Lombok.sneakyThrow(e.getCause());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static CacheControl toCacheControl(String headerValue) {
|
private static CacheControl toCacheControl(String headerValue) {
|
||||||
@@ -237,28 +265,6 @@ public class HttpGetter {
|
|||||||
return DateUtils.parseStandardDate(headerValue);
|
return DateUtils.parseStandardDate(headerValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
|
|
||||||
if (entity.getContentLength() > maxBytes) {
|
|
||||||
throw new IOException(
|
|
||||||
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)"
|
|
||||||
.formatted(entity.getContentLength(), maxBytes));
|
|
||||||
}
|
|
||||||
|
|
||||||
try (InputStream input = entity.getContent()) {
|
|
||||||
if (input == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes();
|
|
||||||
if (bytes.length > maxBytes) {
|
|
||||||
throw new IOException(
|
|
||||||
"Response size exceeds the maximum allowed size (%s bytes)"
|
|
||||||
.formatted(maxBytes));
|
|
||||||
}
|
|
||||||
return bytes;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) {
|
private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) {
|
||||||
HttpClientCache cacheConfig = config.httpClient().cache();
|
HttpClientCache cacheConfig = config.httpClient().cache();
|
||||||
if (!cacheConfig.enabled()) {
|
if (!cacheConfig.enabled()) {
|
||||||
@@ -274,6 +280,43 @@ public class HttpGetter {
|
|||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
private static class LimitedBytesEntityConsumer extends AbstractBinAsyncEntityConsumer<byte[]> {
|
||||||
|
|
||||||
|
private final long maxBytes;
|
||||||
|
private final ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void streamStart(ContentType contentType) {}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected int capacityIncrement() {
|
||||||
|
return Integer.MAX_VALUE;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void data(ByteBuffer src, boolean endOfStream) throws IOException {
|
||||||
|
if (out.size() + src.remaining() > maxBytes) {
|
||||||
|
throw new IOException(
|
||||||
|
"Response size exceeds the maximum allowed size (%s bytes)"
|
||||||
|
.formatted(maxBytes));
|
||||||
|
}
|
||||||
|
byte[] chunk = new byte[src.remaining()];
|
||||||
|
src.get(chunk);
|
||||||
|
out.write(chunk, 0, chunk.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected byte[] generateContent() {
|
||||||
|
return out.toByteArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void releaseResources() {
|
||||||
|
out.reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static class SchemeNotAllowedException extends Exception {
|
public static class SchemeNotAllowedException extends Exception {
|
||||||
private static final long serialVersionUID = 1L;
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
@@ -336,15 +379,15 @@ public class HttpGetter {
|
|||||||
return new HttpRequestBuilder().url(url);
|
return new HttpRequestBuilder().url(url);
|
||||||
}
|
}
|
||||||
|
|
||||||
public ClassicHttpRequest toClassicHttpRequest() {
|
public SimpleHttpRequest toSimpleHttpRequest() {
|
||||||
ClassicHttpRequest req = ClassicRequestBuilder.get(url).build();
|
SimpleRequestBuilder builder = SimpleRequestBuilder.get(url);
|
||||||
if (lastModified != null) {
|
if (lastModified != null) {
|
||||||
req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
|
builder.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
|
||||||
}
|
}
|
||||||
if (eTag != null) {
|
if (eTag != null) {
|
||||||
req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
|
builder.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
|
||||||
}
|
}
|
||||||
return req;
|
return builder.build();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,26 +16,28 @@ import jakarta.inject.Singleton;
|
|||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.apache.hc.client5.http.classic.methods.HttpPost;
|
import org.apache.hc.client5.http.async.methods.SimpleHttpRequest;
|
||||||
|
import org.apache.hc.client5.http.async.methods.SimpleHttpResponse;
|
||||||
|
import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder;
|
||||||
import org.apache.hc.client5.http.config.RequestConfig;
|
import org.apache.hc.client5.http.config.RequestConfig;
|
||||||
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
|
import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
|
||||||
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
|
|
||||||
import org.apache.hc.core5.http.ContentType;
|
import org.apache.hc.core5.http.ContentType;
|
||||||
import org.apache.hc.core5.http.NameValuePair;
|
import org.apache.hc.core5.http.NameValuePair;
|
||||||
import org.apache.hc.core5.http.io.entity.StringEntity;
|
|
||||||
import org.apache.hc.core5.http.message.BasicNameValuePair;
|
import org.apache.hc.core5.http.message.BasicNameValuePair;
|
||||||
|
import org.apache.hc.core5.net.WWWFormCodec;
|
||||||
import org.apache.hc.core5.util.Timeout;
|
import org.apache.hc.core5.util.Timeout;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.concurrent.ExecutionException;
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class PushNotificationService {
|
public class PushNotificationService {
|
||||||
|
|
||||||
private final CloseableHttpClient httpClient;
|
private final CloseableHttpAsyncClient httpClient;
|
||||||
private final Meter meter;
|
private final Meter meter;
|
||||||
private final CommaFeedConfiguration config;
|
private final CommaFeedConfiguration config;
|
||||||
|
|
||||||
@@ -97,13 +99,13 @@ public class PushNotificationService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
HttpPost request = new HttpPost(serverUrl + "/" + topic);
|
SimpleRequestBuilder request = SimpleRequestBuilder.post(serverUrl + "/" + topic);
|
||||||
request.setConfig(
|
request.setRequestConfig(
|
||||||
RequestConfig.custom()
|
RequestConfig.custom()
|
||||||
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
|
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
|
||||||
.build());
|
.build());
|
||||||
request.addHeader("Title", feedTitle);
|
request.addHeader("Title", feedTitle);
|
||||||
request.setEntity(new StringEntity(entryTitle, StandardCharsets.UTF_8));
|
request.setBody(entryTitle, ContentType.TEXT_PLAIN.withCharset(StandardCharsets.UTF_8));
|
||||||
|
|
||||||
if (StringUtils.isNotBlank(entryUrl)) {
|
if (StringUtils.isNotBlank(entryUrl)) {
|
||||||
request.addHeader("Click", entryUrl);
|
request.addHeader("Click", entryUrl);
|
||||||
@@ -113,15 +115,7 @@ public class PushNotificationService {
|
|||||||
request.addHeader("Authorization", "Bearer " + settings.getUserSecret());
|
request.addHeader("Authorization", "Bearer " + settings.getUserSecret());
|
||||||
}
|
}
|
||||||
|
|
||||||
httpClient.execute(
|
send("ntfy", request.build());
|
||||||
request,
|
|
||||||
response -> {
|
|
||||||
if (response.getCode() >= 400) {
|
|
||||||
throw new PushNotificationException(
|
|
||||||
"ntfy notification failed with status " + response.getCode());
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void sendGotify(
|
private void sendGotify(
|
||||||
@@ -152,23 +146,15 @@ public class PushNotificationService {
|
|||||||
.put("click", new JsonObject().put("url", entryUrl))));
|
.put("click", new JsonObject().put("url", entryUrl))));
|
||||||
}
|
}
|
||||||
|
|
||||||
HttpPost request = new HttpPost(serverUrl + "/message");
|
SimpleRequestBuilder request = SimpleRequestBuilder.post(serverUrl + "/message");
|
||||||
request.setConfig(
|
request.setRequestConfig(
|
||||||
RequestConfig.custom()
|
RequestConfig.custom()
|
||||||
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
|
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
|
||||||
.build());
|
.build());
|
||||||
request.addHeader("X-Gotify-Key", token);
|
request.addHeader("X-Gotify-Key", token);
|
||||||
request.setEntity(new StringEntity(json.toString(), ContentType.APPLICATION_JSON));
|
request.setBody(json.toString(), ContentType.APPLICATION_JSON);
|
||||||
|
|
||||||
httpClient.execute(
|
send("gotify", request.build());
|
||||||
request,
|
|
||||||
response -> {
|
|
||||||
if (response.getCode() >= 400) {
|
|
||||||
throw new PushNotificationException(
|
|
||||||
"gotify notification failed with status " + response.getCode());
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void sendPushover(
|
private void sendPushover(
|
||||||
@@ -194,22 +180,32 @@ public class PushNotificationService {
|
|||||||
params.add(new BasicNameValuePair("url", entryUrl));
|
params.add(new BasicNameValuePair("url", entryUrl));
|
||||||
}
|
}
|
||||||
|
|
||||||
HttpPost request = new HttpPost("https://api.pushover.net/1/messages.json");
|
SimpleRequestBuilder request =
|
||||||
request.setConfig(
|
SimpleRequestBuilder.post("https://api.pushover.net/1/messages.json");
|
||||||
|
request.setRequestConfig(
|
||||||
RequestConfig.custom()
|
RequestConfig.custom()
|
||||||
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
|
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
|
||||||
.build());
|
.build());
|
||||||
request.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));
|
request.setBody(
|
||||||
|
WWWFormCodec.format(params, StandardCharsets.UTF_8),
|
||||||
|
ContentType.APPLICATION_FORM_URLENCODED.withCharset(StandardCharsets.UTF_8));
|
||||||
|
|
||||||
httpClient.execute(
|
send("pushover", request.build());
|
||||||
request,
|
}
|
||||||
response -> {
|
|
||||||
if (response.getCode() >= 400) {
|
private void send(String name, SimpleHttpRequest request) throws IOException {
|
||||||
throw new PushNotificationException(
|
try {
|
||||||
"pushover notification failed with status " + response.getCode());
|
SimpleHttpResponse response = httpClient.execute(request, null).get();
|
||||||
}
|
if (response.getCode() >= 400) {
|
||||||
return null;
|
throw new PushNotificationException(
|
||||||
});
|
name + " notification failed with status " + response.getCode());
|
||||||
|
}
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
throw new IOException("interrupted while sending " + name + " notification", e);
|
||||||
|
} catch (ExecutionException e) {
|
||||||
|
throw new IOException(e.getCause());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class PushNotificationException extends RuntimeException {
|
public static class PushNotificationException extends RuntimeException {
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ import org.mockserver.model.Delay;
|
|||||||
import org.mockserver.model.HttpRequest;
|
import org.mockserver.model.HttpRequest;
|
||||||
import org.mockserver.model.HttpResponse;
|
import org.mockserver.model.HttpResponse;
|
||||||
import org.mockserver.model.MediaType;
|
import org.mockserver.model.MediaType;
|
||||||
|
import org.mockserver.model.Protocol;
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
@@ -359,8 +360,7 @@ class HttpGetterTest {
|
|||||||
|
|
||||||
IOException e = Assertions.assertThrows(IOException.class, () -> getter.get(this.feedUrl));
|
IOException e = Assertions.assertThrows(IOException.class, () -> getter.get(this.feedUrl));
|
||||||
Assertions.assertEquals(
|
Assertions.assertEquals(
|
||||||
"Response size (100000 bytes) exceeds the maximum allowed size (10000 bytes)",
|
"Response size exceeds the maximum allowed size (10000 bytes)", e.getMessage());
|
||||||
e.getMessage());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -416,7 +416,7 @@ class HttpGetterTest {
|
|||||||
@Nested
|
@Nested
|
||||||
class Compression {
|
class Compression {
|
||||||
|
|
||||||
private static final String ACCEPT_ENCODING = "gzip, deflate, br";
|
private static final String ACCEPT_ENCODING = "deflate, gzip";
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void gzip() throws Exception {
|
void gzip() throws Exception {
|
||||||
@@ -590,4 +590,30 @@ class HttpGetterTest {
|
|||||||
UnknownHostException.class, () -> getter.get("http://[::a00:1]")); // 10.0.0.1
|
UnknownHostException.class, () -> getter.get("http://[::a00:1]")); // 10.0.0.1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Nested
|
||||||
|
class Protocols {
|
||||||
|
@Test
|
||||||
|
void usesHttp2OverTls() throws Exception {
|
||||||
|
mockServerClient
|
||||||
|
.when(HttpRequest.request().withMethod("GET").withProtocol(Protocol.HTTP_2))
|
||||||
|
.respond(HttpResponse.response().withBody("h2"));
|
||||||
|
mockServerClient
|
||||||
|
.when(HttpRequest.request().withMethod("GET").withProtocol(Protocol.HTTP_1_1))
|
||||||
|
.respond(HttpResponse.response().withBody("h1"));
|
||||||
|
|
||||||
|
HttpResult result = getter.get("https://localhost:" + mockServerClient.getPort());
|
||||||
|
Assertions.assertEquals("h2", new String(result.content()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void fallsBackToHttp1OverCleartext() throws Exception {
|
||||||
|
mockServerClient
|
||||||
|
.when(HttpRequest.request().withMethod("GET").withProtocol(Protocol.HTTP_1_1))
|
||||||
|
.respond(HttpResponse.response().withBody("h1"));
|
||||||
|
|
||||||
|
HttpResult result = getter.get("http://localhost:" + mockServerClient.getPort());
|
||||||
|
Assertions.assertEquals("h1", new String(result.content()));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user