Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-03-21 21:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

update to httpclient5

Browse Source
This commit is contained in:
Athou
2023-12-25 19:41:14 +01:00
parent 11aff68052
commit 5ba248eaba
3 changed files with 86 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
commafeed-server/pom.xml
View File

@@ -392,9 +392,13 @@
<artifactId>gwt-servlet</artifactId> <artifactId>gwt-servlet</artifactId>
<version>2.10.0</version> <version>2.10.0</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
</dependency>
<dependency> <dependency>
<groupId>io.github.hakky54</groupId> <groupId>io.github.hakky54</groupId>
<artifactId>sslcontext-kickstart</artifactId> <artifactId>sslcontext-kickstart-for-apache5</artifactId>
<version>8.2.0</version> <version>8.2.0</version>
</dependency> </dependency>

123
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -1,23 +1,30 @@
package com.commafeed.backend; package com.commafeed.backend;
import java.io.IOException; import java.io.IOException;
import java.net.URI;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.TimeUnit;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.http.Header; import org.apache.hc.client5.http.config.ConnectionConfig;
import org.apache.http.HttpEntity; import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.http.NameValuePair; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.http.client.config.RequestConfig; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.http.client.methods.HttpGet; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.http.client.protocol.HttpClientContext; import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.hc.client5.http.protocol.RedirectLocations;
import org.apache.http.impl.client.HttpClientBuilder; import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.http.message.BasicHeader; import org.apache.hc.core5.http.Header;
import org.apache.http.util.EntityUtils; import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.http.HttpStatus;
import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration;
@@ -30,6 +37,7 @@ import jakarta.inject.Singleton;
import lombok.Getter; import lombok.Getter;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import nl.altindag.ssl.SSLFactory; import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
/** /**
* Smart HTTP getter: handles gzip, ssl, last modified and etag headers * Smart HTTP getter: handles gzip, ssl, last modified and etag headers
@@ -47,7 +55,7 @@ public class HttpGetter implements Managed {
this.client = newClient(userAgent, config.getApplicationSettings().getBackgroundThreads()); this.client = newClient(userAgent, config.getApplicationSettings().getBackgroundThreads());
} }
public HttpResult getBinary(String url, int timeout) throws IOException, NotModifiedException, InterruptedException { public HttpResult getBinary(String url, int timeout) throws IOException, NotModifiedException {
return getBinary(url, null, null, timeout); return getBinary(url, null, null, timeout);
} }
@@ -62,11 +70,10 @@ public class HttpGetter implements Managed {
* @throws NotModifiedException * @throws NotModifiedException
* if the url hasn't changed since we asked for it last time * if the url hasn't changed since we asked for it last time
*/ */
public HttpResult getBinary(String url, String lastModified, String eTag, int timeout) public HttpResult getBinary(String url, String lastModified, String eTag, int timeout) throws IOException, NotModifiedException {
throws IOException, NotModifiedException, InterruptedException {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
HttpGet request = new HttpGet(url); ClassicHttpRequest request = ClassicRequestBuilder.get(url).build();
if (lastModified != null) { if (lastModified != null) {
request.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified); request.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
} }
@@ -75,42 +82,50 @@ public class HttpGetter implements Managed {
} }
HttpClientContext context = HttpClientContext.create(); HttpClientContext context = HttpClientContext.create();
context.setRequestConfig( context.setRequestConfig(RequestConfig.custom().setResponseTimeout(timeout, TimeUnit.MILLISECONDS).build());
RequestConfig.custom().setConnectTimeout(timeout).setConnectionRequestTimeout(timeout).setSocketTimeout(timeout).build());
try (CloseableHttpResponse response = client.execute(request, context)) { HttpResponse response = client.execute(request, context, resp -> {
int code = response.getStatusLine().getStatusCode(); int code = resp.getCode();
if (code == HttpStatus.NOT_MODIFIED_304) { String lastModifiedHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
throw new NotModifiedException("'304 - not modified' http code received");
} else if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
String lastModifiedHeader = Optional.ofNullable(response.getFirstHeader(HttpHeaders.LAST_MODIFIED))
.map(NameValuePair::getValue) .map(NameValuePair::getValue)
.map(StringUtils::trimToNull) .map(StringUtils::trimToNull)
.orElse(null); .orElse(null);
if (lastModifiedHeader != null && lastModifiedHeader.equals(lastModified)) { String eTagHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
throw new NotModifiedException("lastModifiedHeader is the same");
}
String eTagHeader = Optional.ofNullable(response.getFirstHeader(HttpHeaders.ETAG))
.map(NameValuePair::getValue) .map(NameValuePair::getValue)
.map(StringUtils::trimToNull) .map(StringUtils::trimToNull)
.orElse(null); .orElse(null);
if (eTagHeader != null && eTagHeader.equals(eTag)) {
throw new NotModifiedException("eTagHeader is the same");
}
HttpEntity entity = response.getEntity(); byte[] content = resp.getEntity() == null ? null : EntityUtils.toByteArray(resp.getEntity());
byte[] content = entity == null ? null : EntityUtils.toByteArray(entity); String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null);
String contentType = Optional.ofNullable(entity).map(HttpEntity::getContentType).map(Header::getValue).orElse(null); String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations())
String urlAfterRedirect = CollectionUtils.isEmpty(context.getRedirectLocations()) ? url .map(RedirectLocations::getAll)
: Iterables.getLast(context.getRedirectLocations()).toString(); .map(l -> Iterables.getLast(l, null))
.map(URI::toString)
.orElse(url);
long duration = System.currentTimeMillis() - start; return new HttpResponse(code, lastModifiedHeader, eTagHeader, content, contentType, urlAfterRedirect);
return new HttpResult(content, contentType, lastModifiedHeader, eTagHeader, duration, urlAfterRedirect); });
int code = response.getCode();
if (code == HttpStatus.NOT_MODIFIED_304) {
throw new NotModifiedException("'304 - not modified' http code received");
} else if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
} }
String lastModifiedHeader = response.getLastModifiedHeader();
if (lastModifiedHeader != null && lastModifiedHeader.equals(lastModified)) {
throw new NotModifiedException("lastModifiedHeader is the same");
}
String eTagHeader = response.getETagHeader();
if (eTagHeader != null && eTagHeader.equals(eTag)) {
throw new NotModifiedException("eTagHeader is the same");
}
long duration = System.currentTimeMillis() - start;
return new HttpResult(response.getContent(), response.getContentType(), lastModifiedHeader, eTagHeader, duration,
response.getUrlAfterRedirect());
} }
private CloseableHttpClient newClient(String userAgent, int poolSize) { private CloseableHttpClient newClient(String userAgent, int poolSize) {
@@ -121,16 +136,21 @@ public class HttpGetter implements Managed {
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache")); headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache")); headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
PoolingHttpClientConnectionManager connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
.setSSLSocketFactory(Apache5SslUtils.toSocketFactory(sslFactory))
.setDefaultConnectionConfig(
ConnectionConfig.custom().setConnectTimeout(Timeout.ofSeconds(5)).setTimeToLive(TimeValue.ofSeconds(30)).build())
.setMaxConnPerRoute(poolSize)
.setMaxConnTotal(poolSize)
.build();
return HttpClientBuilder.create() return HttpClientBuilder.create()
.useSystemProperties() .useSystemProperties()
.disableAutomaticRetries() .disableAutomaticRetries()
.disableCookieManagement() .disableCookieManagement()
.setUserAgent(userAgent) .setUserAgent(userAgent)
.setDefaultHeaders(headers) .setDefaultHeaders(headers)
.setSSLContext(sslFactory.getSslContext()) .setConnectionManager(connectionManager)
.setSSLHostnameVerifier(sslFactory.getHostnameVerifier())
.setMaxConnTotal(poolSize)
.setMaxConnPerRoute(poolSize)
.build(); .build();
} }
@@ -177,6 +197,17 @@ public class HttpGetter implements Managed {
} }
@Getter
@RequiredArgsConstructor
private static class HttpResponse {
private final int code;
private final String lastModifiedHeader;
private final String eTagHeader;
private final byte[] content;
private final String contentType;
private final String urlAfterRedirect;
}
@Getter @Getter
@RequiredArgsConstructor @RequiredArgsConstructor
public static class HttpResult { public static class HttpResult {

8
commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java
View File

@@ -6,7 +6,7 @@ import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.http.conn.ConnectTimeoutException; import org.apache.hc.client5.http.ConnectTimeoutException;
import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.http.HttpStatus;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
@@ -135,7 +135,7 @@ class HttpGetterTest {
} }
@Test @Test
void lastModifiedReturns304() throws Exception { void lastModifiedReturns304() {
this.mockServerClient.when(HttpRequest.request().withMethod("GET").withHeader(HttpHeaders.IF_MODIFIED_SINCE, "123456")) this.mockServerClient.when(HttpRequest.request().withMethod("GET").withHeader(HttpHeaders.IF_MODIFIED_SINCE, "123456"))
.respond(HttpResponse.response().withStatusCode(HttpStatus.NOT_MODIFIED_304)); .respond(HttpResponse.response().withStatusCode(HttpStatus.NOT_MODIFIED_304));
@@ -143,7 +143,7 @@ class HttpGetterTest {
} }
@Test @Test
void eTagReturns304() throws Exception { void eTagReturns304() {
this.mockServerClient.when(HttpRequest.request().withMethod("GET").withHeader(HttpHeaders.IF_NONE_MATCH, "78910")) this.mockServerClient.when(HttpRequest.request().withMethod("GET").withHeader(HttpHeaders.IF_NONE_MATCH, "78910"))
.respond(HttpResponse.response().withStatusCode(HttpStatus.NOT_MODIFIED_304)); .respond(HttpResponse.response().withStatusCode(HttpStatus.NOT_MODIFIED_304));
@@ -151,7 +151,7 @@ class HttpGetterTest {
} }
@Test @Test
void ignoreCookie() throws Exception { void ignoreCookie() {
AtomicInteger calls = new AtomicInteger(); AtomicInteger calls = new AtomicInteger();
this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(req -> { this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(req -> {