Athou_commafeed/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java

package com.commafeed.backend;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;

import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicHeader;
import org.apache.http.util.EntityUtils;
import org.eclipse.jetty.http.HttpStatus;

import com.commafeed.CommaFeedConfiguration;
import com.google.common.collect.Iterables;
import com.google.common.net.HttpHeaders;

import io.dropwizard.lifecycle.Managed;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import nl.altindag.ssl.SSLFactory;

/**
 * Smart HTTP getter: handles gzip, ssl, last modified and etag headers
 *
 */
@Singleton
public class HttpGetter implements Managed {

	private final CloseableHttpClient client;

	@Inject
	public HttpGetter(CommaFeedConfiguration config) {
		String userAgent = Optional.ofNullable(config.getApplicationSettings().getUserAgent())
				.orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", config.getVersion()));
		this.client = newClient(userAgent, config.getApplicationSettings().getBackgroundThreads());
	}

	public HttpResult getBinary(String url, int timeout) throws IOException, NotModifiedException, InterruptedException {
		return getBinary(url, null, null, timeout);
	}

	/**
	 *
	 * @param url
	 *            the url to retrive
	 * @param lastModified
	 *            header we got last time we queried that url, or null
	 * @param eTag
	 *            header we got last time we queried that url, or null
	 * @throws NotModifiedException
	 *             if the url hasn't changed since we asked for it last time
	 */
	public HttpResult getBinary(String url, String lastModified, String eTag, int timeout)
			throws IOException, NotModifiedException, InterruptedException {
		long start = System.currentTimeMillis();

		HttpGet request = new HttpGet(url);
		if (lastModified != null) {
			request.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
		}
		if (eTag != null) {
			request.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
		}

		HttpClientContext context = HttpClientContext.create();
		context.setRequestConfig(
				RequestConfig.custom().setConnectTimeout(timeout).setConnectionRequestTimeout(timeout).setSocketTimeout(timeout).build());

		try (CloseableHttpResponse response = client.execute(request, context)) {
			int code = response.getStatusLine().getStatusCode();
			if (code == HttpStatus.NOT_MODIFIED_304) {
				throw new NotModifiedException("'304 - not modified' http code received");
			} else if (code >= 300) {
				throw new HttpResponseException(code, "Server returned HTTP error code " + code);
			}

			String lastModifiedHeader = Optional.ofNullable(response.getFirstHeader(HttpHeaders.LAST_MODIFIED))
					.map(NameValuePair::getValue)
					.map(StringUtils::trimToNull)
					.orElse(null);
			if (lastModifiedHeader != null && lastModifiedHeader.equals(lastModified)) {
				throw new NotModifiedException("lastModifiedHeader is the same");
			}

			String eTagHeader = Optional.ofNullable(response.getFirstHeader(HttpHeaders.ETAG))
					.map(NameValuePair::getValue)
					.map(StringUtils::trimToNull)
					.orElse(null);
			if (eTagHeader != null && eTagHeader.equals(eTag)) {
				throw new NotModifiedException("eTagHeader is the same");
			}

			HttpEntity entity = response.getEntity();
			byte[] content = entity == null ? null : EntityUtils.toByteArray(entity);
			String contentType = Optional.ofNullable(entity).map(HttpEntity::getContentType).map(Header::getValue).orElse(null);
			String urlAfterRedirect = CollectionUtils.isEmpty(context.getRedirectLocations()) ? url
					: Iterables.getLast(context.getRedirectLocations()).toString();

			long duration = System.currentTimeMillis() - start;
			return new HttpResult(content, contentType, lastModifiedHeader, eTagHeader, duration, urlAfterRedirect);
		}
	}

	private CloseableHttpClient newClient(String userAgent, int poolSize) {
		SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();

		List<Header> headers = new ArrayList<>();
		headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
		headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
		headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));

		return HttpClientBuilder.create()
				.useSystemProperties()
				.disableAutomaticRetries()
				.disableCookieManagement()
				.setUserAgent(userAgent)
				.setDefaultHeaders(headers)
				.setSSLContext(sslFactory.getSslContext())
				.setSSLHostnameVerifier(sslFactory.getHostnameVerifier())
				.setMaxConnTotal(poolSize)
				.setMaxConnPerRoute(poolSize)
				.build();
	}

	@Override
	public void stop() throws Exception {
		client.close();
	}

	@Getter
	public static class NotModifiedException extends Exception {
		private static final long serialVersionUID = 1L;

		/**
		 * if the value of this header changed, this is its new value
		 */
		private final String newLastModifiedHeader;

		/**
		 * if the value of this header changed, this is its new value
		 */
		private final String newEtagHeader;

		public NotModifiedException(String message) {
			this(message, null, null);
		}

		public NotModifiedException(String message, String newLastModifiedHeader, String newEtagHeader) {
			super(message);
			this.newLastModifiedHeader = newLastModifiedHeader;
			this.newEtagHeader = newEtagHeader;
		}
	}

	@Getter
	public static class HttpResponseException extends IOException {
		private static final long serialVersionUID = 1L;

		private final int code;

		public HttpResponseException(int code, String message) {
			super(message);
			this.code = code;
		}

	}

	@Getter
	@RequiredArgsConstructor
	public static class HttpResult {
		private final byte[] content;
		private final String contentType;
		private final String lastModifiedSince;
		private final String eTag;
		private final long duration;
		private final String urlAfterRedirect;
	}

}