garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

set different http timeouts based on what we are fetching

Browse Source
This commit is contained in:
Athou
2013-06-29 12:05:22 +02:00
parent fd83609d68
commit a425c33cf8
5 changed files with 16 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -59,9 +59,9 @@ public class HttpGetter {
private static final X509HostnameVerifier VERIFIER = new DefaultHostnameVerifier();
public HttpResult getBinary(String url) throws ClientProtocolException,
public HttpResult getBinary(String url, int timeout) throws ClientProtocolException,
IOException, NotModifiedException {
return getBinary(url, null, null);
return getBinary(url, null, null, timeout);
}
/**
@@ -78,12 +78,12 @@ public class HttpGetter {
* @throws NotModifiedException
* if the url hasn't changed since we asked for it last time
*/
public HttpResult getBinary(String url, String lastModified, String eTag)
public HttpResult getBinary(String url, String lastModified, String eTag, int timeout)
throws ClientProtocolException, IOException, NotModifiedException {
HttpResult result = null;
long start = System.currentTimeMillis();
HttpClient client = newClient();
HttpClient client = newClient(timeout);
try {
HttpGet httpget = new HttpGet(url);
httpget.addHeader(HttpHeaders.ACCEPT_LANGUAGE, "en");
@@ -191,7 +191,7 @@ public class HttpGetter {
}
public static HttpClient newClient() {
public static HttpClient newClient(int timeout) {
DefaultHttpClient client = new SystemDefaultHttpClient();
SSLSocketFactory ssf = new SSLSocketFactory(SSL_CONTEXT, VERIFIER);
@@ -202,8 +202,8 @@ public class HttpGetter {
HttpParams params = client.getParams();
HttpClientParams.setCookiePolicy(params, CookiePolicy.IGNORE_COOKIES);
HttpProtocolParams.setContentCharset(params, "UTF-8");
HttpConnectionParams.setConnectionTimeout(params, 20000);
HttpConnectionParams.setSoTimeout(params, 20000);
HttpConnectionParams.setConnectionTimeout(params, timeout);
HttpConnectionParams.setSoTimeout(params, timeout);
client.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0,
false));
return new DecompressingHttpClient(client);

7
src/main/java/com/commafeed/backend/feeds/FaviconFetcher.java
View File

@@ -25,6 +25,7 @@ public class FaviconFetcher {
private static long MIN_ICON_LENGTH = 100;
private static long MAX_ICON_LENGTH = 20000;
private static int TIMEOUT = 4000;
protected static List<String> ICON_MIMETYPES = Arrays.asList(
"image/x-icon", "image/vnd.microsoft.icon", "image/ico",
@@ -70,7 +71,7 @@ public class FaviconFetcher {
try {
url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url);
HttpResult result = getter.getBinary(url);
HttpResult result = getter.getBinary(url, TIMEOUT);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
@@ -118,7 +119,7 @@ public class FaviconFetcher {
Document doc = null;
try {
HttpResult result = getter.getBinary(url);
HttpResult result = getter.getBinary(url, TIMEOUT);
doc = Jsoup.parse(new String(result.getContent()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon");
@@ -144,7 +145,7 @@ public class FaviconFetcher {
byte[] bytes = null;
String contentType = null;
try {
HttpResult result = getter.getBinary(href);
HttpResult result = getter.getBinary(href, TIMEOUT);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {

5
src/main/java/com/commafeed/backend/feeds/FeedFetcher.java
View File

@@ -37,12 +37,13 @@ public class FeedFetcher {
log.debug("Fetching feed {}", feedUrl);
FetchedFeed fetchedFeed = null;
HttpResult result = getter.getBinary(feedUrl, lastModified, eTag);
int timeout = 20000;
HttpResult result = getter.getBinary(feedUrl, lastModified, eTag, timeout);
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(
StringUtils.newStringUtf8(result.getContent()), feedUrl);
if (org.apache.commons.lang.StringUtils.isNotBlank(extractedUrl)) {
result = getter.getBinary(extractedUrl, lastModified, eTag);
result = getter.getBinary(extractedUrl, lastModified, eTag, timeout);
feedUrl = extractedUrl;
}
}

2
src/main/java/com/commafeed/backend/pubsubhubbub/SubscriptionHandler.java
View File

@@ -68,7 +68,7 @@ public class SubscriptionHandler {
post.setHeader(HttpHeaders.CONTENT_TYPE,
MediaType.APPLICATION_FORM_URLENCODED);
HttpClient client = HttpGetter.newClient();
HttpClient client = HttpGetter.newClient(20000);
try {
post.setEntity(new UrlEncodedFormEntity(nvp));
HttpResponse response = client.execute(post);

2
src/main/java/com/commafeed/frontend/rest/resources/ServerREST.java
View File

@@ -40,7 +40,7 @@ public class ServerREST extends AbstractResourceREST {
url = FeedUtils.imageProxyDecoder(url);
try {
HttpResult result = httpGetter.getBinary(url);
HttpResult result = httpGetter.getBinary(url, 20000);
return Response.ok(result.getContent()).build();
} catch (Exception e) {
return Response.status(Status.SERVICE_UNAVAILABLE)