garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

httpclient upgrade

Browse Source
This commit is contained in:
Athou
2013-11-05 15:10:09 +01:00
parent 52df661238
commit 05036778d6
3 changed files with 43 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@@ -309,7 +309,7 @@
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId> <artifactId>httpclient</artifactId>
<version>4.2.5</version> <version>4.3.1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.jsoup</groupId> <groupId>org.jsoup</groupId>

99
src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -7,44 +7,33 @@ import java.security.cert.X509Certificate;
import javax.net.ssl.KeyManager; import javax.net.ssl.KeyManager;
import javax.net.ssl.SSLContext; import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLException;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager; import javax.net.ssl.X509TrustManager;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.http.Consts;
import org.apache.http.Header; import org.apache.http.Header;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders; import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus; import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException; import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.HttpResponseException; import org.apache.http.client.HttpResponseException;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.params.CookiePolicy; import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.params.HttpClientParams; import org.apache.http.config.ConnectionConfig;
import org.apache.http.conn.ClientConnectionManager; import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.scheme.Scheme; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.conn.scheme.SchemeRegistry; import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.conn.ssl.SSLSocketFactory; import org.apache.http.impl.client.HttpClients;
import org.apache.http.conn.ssl.X509HostnameVerifier;
import org.apache.http.impl.client.DecompressingHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.SystemDefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.ExecutionContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.apache.wicket.util.io.IOUtils;
/** /**
* Smart HTTP getter: handles gzip, ssl, last modified and etag headers * Smart HTTP getter: handles gzip, ssl, last modified and etag headers
@@ -57,8 +46,6 @@ public class HttpGetter {
private static final String ACCEPT_LANGUAGE = "en"; private static final String ACCEPT_LANGUAGE = "en";
private static final String PRAGMA_NO_CACHE = "No-cache"; private static final String PRAGMA_NO_CACHE = "No-cache";
private static final String CACHE_CONTROL_NO_CACHE = "no-cache"; private static final String CACHE_CONTROL_NO_CACHE = "no-cache";
private static final String UTF8 = "UTF-8";
private static final String HTTPS = "https";
private static SSLContext SSL_CONTEXT = null; private static SSLContext SSL_CONTEXT = null;
static { static {
@@ -70,8 +57,6 @@ public class HttpGetter {
} }
} }
private static final X509HostnameVerifier VERIFIER = new DefaultHostnameVerifier();
public HttpResult getBinary(String url, int timeout) throws ClientProtocolException, IOException, NotModifiedException { public HttpResult getBinary(String url, int timeout) throws ClientProtocolException, IOException, NotModifiedException {
return getBinary(url, null, null, timeout); return getBinary(url, null, null, timeout);
} }
@@ -95,10 +80,11 @@ public class HttpGetter {
HttpResult result = null; HttpResult result = null;
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
HttpClient client = newClient(timeout); CloseableHttpClient client = newClient(timeout);
CloseableHttpResponse response = null;
try { try {
HttpGet httpget = new HttpGet(url); HttpGet httpget = new HttpGet(url);
HttpContext context = new BasicHttpContext(); HttpClientContext context = HttpClientContext.create();
httpget.addHeader(HttpHeaders.ACCEPT_LANGUAGE, ACCEPT_LANGUAGE); httpget.addHeader(HttpHeaders.ACCEPT_LANGUAGE, ACCEPT_LANGUAGE);
httpget.addHeader(HttpHeaders.PRAGMA, PRAGMA_NO_CACHE); httpget.addHeader(HttpHeaders.PRAGMA, PRAGMA_NO_CACHE);
@@ -112,7 +98,6 @@ public class HttpGetter {
httpget.addHeader(HttpHeaders.IF_NONE_MATCH, eTag); httpget.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
} }
HttpResponse response = null;
try { try {
response = client.execute(httpget, context); response = client.execute(httpget, context);
int code = response.getStatusLine().getStatusCode(); int code = response.getStatusLine().getStatusCode();
@@ -150,14 +135,15 @@ public class HttpGetter {
contentType = entity.getContentType().getValue(); contentType = entity.getContentType().getValue();
} }
} }
HttpUriRequest req = (HttpUriRequest) context.getAttribute(ExecutionContext.HTTP_REQUEST); HttpUriRequest req = (HttpUriRequest) context.getRequest();
HttpHost host = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST); HttpHost host = context.getTargetHost();
String urlAfterRedirect = req.getURI().isAbsolute() ? req.getURI().toString() : host.toURI() + req.getURI(); String urlAfterRedirect = req.getURI().isAbsolute() ? req.getURI().toString() : host.toURI() + req.getURI();
long duration = System.currentTimeMillis() - start; long duration = System.currentTimeMillis() - start;
result = new HttpResult(content, contentType, lastModifiedHeaderValue, eTagHeaderValue, duration, urlAfterRedirect); result = new HttpResult(content, contentType, lastModifiedHeaderValue, eTagHeaderValue, duration, urlAfterRedirect);
} finally { } finally {
client.getConnectionManager().shutdown(); IOUtils.closeQuietly(response);
IOUtils.closeQuietly(client);
} }
return result; return result;
} }
@@ -205,21 +191,24 @@ public class HttpGetter {
} }
} }
public static HttpClient newClient(int timeout) { public static CloseableHttpClient newClient(int timeout) {
DefaultHttpClient client = new SystemDefaultHttpClient(); HttpClientBuilder builder = HttpClients.custom();
builder.useSystemProperties();
builder.disableAutomaticRetries();
SSLSocketFactory ssf = new SSLSocketFactory(SSL_CONTEXT, VERIFIER); builder.setSslcontext(SSL_CONTEXT);
ClientConnectionManager ccm = client.getConnectionManager(); builder.setHostnameVerifier(SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
SchemeRegistry sr = ccm.getSchemeRegistry();
sr.register(new Scheme(HTTPS, 443, ssf));
HttpParams params = client.getParams(); RequestConfig.Builder configBuilder = RequestConfig.custom();
HttpClientParams.setCookiePolicy(params, CookiePolicy.IGNORE_COOKIES); configBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
HttpProtocolParams.setContentCharset(params, UTF8); configBuilder.setSocketTimeout(timeout);
HttpConnectionParams.setConnectionTimeout(params, timeout); configBuilder.setConnectTimeout(timeout);
HttpConnectionParams.setSoTimeout(params, timeout); configBuilder.setConnectionRequestTimeout(timeout);
client.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); builder.setDefaultRequestConfig(configBuilder.build());
return new DecompressingHttpClient(client);
builder.setDefaultConnectionConfig(ConnectionConfig.custom().setCharset(Consts.UTF_8).build());
return builder.build();
} }
public static class NotModifiedException extends Exception { public static class NotModifiedException extends Exception {
@@ -245,24 +234,4 @@ public class HttpGetter {
return null; return null;
} }
} }
private static class DefaultHostnameVerifier implements X509HostnameVerifier {
@Override
public void verify(String string, SSLSocket ssls) throws IOException {
}
@Override
public void verify(String string, X509Certificate xc) throws SSLException {
}
@Override
public void verify(String string, String[] strings, String[] strings1) throws SSLException {
}
@Override
public boolean verify(String string, SSLSession ssls) {
return true;
}
};
} }

13
src/main/java/com/commafeed/backend/pubsubhubbub/SubscriptionHandler.java
View File

@@ -9,13 +9,14 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpHeaders; import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair; import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair; import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.apache.wicket.util.io.IOUtils;
import com.commafeed.backend.HttpGetter; import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.feeds.FeedRefreshTaskGiver; import com.commafeed.backend.feeds.FeedRefreshTaskGiver;
@@ -67,10 +68,11 @@ public class SubscriptionHandler {
post.setHeader(HttpHeaders.USER_AGENT, "CommaFeed"); post.setHeader(HttpHeaders.USER_AGENT, "CommaFeed");
post.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_FORM_URLENCODED); post.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_FORM_URLENCODED);
HttpClient client = HttpGetter.newClient(20000); CloseableHttpClient client = HttpGetter.newClient(20000);
CloseableHttpResponse response = null;
try { try {
post.setEntity(new UrlEncodedFormEntity(nvp)); post.setEntity(new UrlEncodedFormEntity(nvp));
HttpResponse response = client.execute(post); response = client.execute(post);
int code = response.getStatusLine().getStatusCode(); int code = response.getStatusLine().getStatusCode();
if (code != 204 && code != 202 && code != 200) { if (code != 204 && code != 202 && code != 200) {
@@ -90,7 +92,8 @@ public class SubscriptionHandler {
} catch (Exception e) { } catch (Exception e) {
log.error("Could not subscribe to {} for {} : " + e.getMessage(), hub, topic); log.error("Could not subscribe to {} for {} : " + e.getMessage(), hub, topic);
} finally { } finally {
client.getConnectionManager().shutdown(); IOUtils.closeQuietly(response);
IOUtils.closeQuietly(client);
} }
} }
} }