package com.commafeed.backend; import java.io.IOException; import java.net.SocketTimeoutException; import java.util.Arrays; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.IOUtils; import org.apache.hc.client5.http.ConnectTimeoutException; import org.eclipse.jetty.http.HttpStatus; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.mockserver.client.MockServerClient; import org.mockserver.junit.jupiter.MockServerExtension; import org.mockserver.model.Delay; import org.mockserver.model.HttpRequest; import org.mockserver.model.HttpResponse; import org.mockserver.model.MediaType; import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration.ApplicationSettings; import com.commafeed.backend.HttpGetter.HttpResponseException; import com.commafeed.backend.HttpGetter.HttpResult; import com.commafeed.backend.HttpGetter.NotModifiedException; import com.google.common.net.HttpHeaders; import io.dropwizard.util.DataSize; @ExtendWith(MockServerExtension.class) class HttpGetterTest { private static final int TIMEOUT = 10000; private MockServerClient mockServerClient; private String feedUrl; private byte[] feedContent; private HttpGetter getter; @BeforeEach void init(MockServerClient mockServerClient) throws IOException { this.mockServerClient = mockServerClient; this.mockServerClient.reset(); this.feedUrl = "http://localhost:" + this.mockServerClient.getPort() + "/"; this.feedContent = IOUtils.toByteArray(Objects.requireNonNull(getClass().getResource("/feed/rss.xml"))); ApplicationSettings settings = new ApplicationSettings(); settings.setUserAgent("http-getter-test"); settings.setBackgroundThreads(3); settings.setMaxFeedResponseSize(DataSize.kilobytes(1)); CommaFeedConfiguration config = new CommaFeedConfiguration(); config.setApplicationSettings(settings); this.getter = new HttpGetter(config); } @ParameterizedTest @ValueSource( ints = { HttpStatus.UNAUTHORIZED_401, HttpStatus.FORBIDDEN_403, HttpStatus.NOT_FOUND_404, HttpStatus.INTERNAL_SERVER_ERROR_500 }) void errorCodes(int code) { this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(HttpResponse.response().withStatusCode(code)); HttpResponseException e = Assertions.assertThrows(HttpResponseException.class, () -> getter.getBinary(this.feedUrl, TIMEOUT)); Assertions.assertEquals(code, e.getCode()); } @Test void validFeed() throws Exception { this.mockServerClient.when(HttpRequest.request().withMethod("GET")) .respond(HttpResponse.response() .withBody(feedContent) .withContentType(MediaType.APPLICATION_ATOM_XML) .withHeader(HttpHeaders.LAST_MODIFIED, "123456") .withHeader(HttpHeaders.ETAG, "78910")); HttpResult result = getter.getBinary(this.feedUrl, TIMEOUT); Assertions.assertArrayEquals(feedContent, result.getContent()); Assertions.assertEquals(MediaType.APPLICATION_ATOM_XML.toString(), result.getContentType()); Assertions.assertEquals("123456", result.getLastModifiedSince()); Assertions.assertEquals("78910", result.getETag()); Assertions.assertTrue(result.getDuration() > 0); Assertions.assertEquals(this.feedUrl, result.getUrlAfterRedirect()); } @ParameterizedTest @ValueSource( ints = { HttpStatus.MOVED_PERMANENTLY_301, HttpStatus.MOVED_TEMPORARILY_302, HttpStatus.TEMPORARY_REDIRECT_307, HttpStatus.PERMANENT_REDIRECT_308 }) void followRedirects(int code) throws Exception { // first redirect this.mockServerClient.when(HttpRequest.request().withMethod("GET").withPath("/")) .respond(HttpResponse.response() .withStatusCode(code) .withHeader(HttpHeaders.LOCATION, "http://localhost:" + this.mockServerClient.getPort() + "/redirected")); // second redirect this.mockServerClient.when(HttpRequest.request().withMethod("GET").withPath("/redirected")) .respond(HttpResponse.response() .withStatusCode(code) .withHeader(HttpHeaders.LOCATION, "http://localhost:" + this.mockServerClient.getPort() + "/redirected-2")); // final destination this.mockServerClient.when(HttpRequest.request().withMethod("GET").withPath("/redirected-2")) .respond(HttpResponse.response().withBody(feedContent).withContentType(MediaType.APPLICATION_ATOM_XML)); HttpResult result = getter.getBinary(this.feedUrl, TIMEOUT); Assertions.assertEquals("http://localhost:" + this.mockServerClient.getPort() + "/redirected-2", result.getUrlAfterRedirect()); } @Test void dataTimeout() { int smallTimeout = 500; this.mockServerClient.when(HttpRequest.request().withMethod("GET")) .respond(HttpResponse.response().withDelay(Delay.milliseconds(smallTimeout * 2))); Assertions.assertThrows(SocketTimeoutException.class, () -> getter.getBinary(this.feedUrl, smallTimeout)); } @Test void connectTimeout() { // try to connect to a non-routable address // https://stackoverflow.com/a/904609/1885506 Assertions.assertThrows(ConnectTimeoutException.class, () -> getter.getBinary("http://10.255.255.1", 2000)); } @Test void userAgent() throws Exception { this.mockServerClient.when(HttpRequest.request().withMethod("GET").withHeader(HttpHeaders.USER_AGENT, "http-getter-test")) .respond(HttpResponse.response().withBody("ok")); HttpResult result = getter.getBinary(this.feedUrl, TIMEOUT); Assertions.assertEquals("ok", new String(result.getContent())); } @Test void lastModifiedReturns304() { this.mockServerClient.when(HttpRequest.request().withMethod("GET").withHeader(HttpHeaders.IF_MODIFIED_SINCE, "123456")) .respond(HttpResponse.response().withStatusCode(HttpStatus.NOT_MODIFIED_304)); Assertions.assertThrows(NotModifiedException.class, () -> getter.getBinary(this.feedUrl, "123456", null, TIMEOUT)); } @Test void eTagReturns304() { this.mockServerClient.when(HttpRequest.request().withMethod("GET").withHeader(HttpHeaders.IF_NONE_MATCH, "78910")) .respond(HttpResponse.response().withStatusCode(HttpStatus.NOT_MODIFIED_304)); Assertions.assertThrows(NotModifiedException.class, () -> getter.getBinary(this.feedUrl, null, "78910", TIMEOUT)); } @Test void ignoreCookie() { AtomicInteger calls = new AtomicInteger(); this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(req -> { calls.incrementAndGet(); if (req.containsHeader(HttpHeaders.COOKIE)) { throw new Exception("cookie should not be sent by the client"); } return HttpResponse.response().withBody("ok").withHeader(HttpHeaders.SET_COOKIE, "foo=bar"); }); Assertions.assertDoesNotThrow(() -> getter.getBinary(this.feedUrl, TIMEOUT)); Assertions.assertDoesNotThrow(() -> getter.getBinary(this.feedUrl, TIMEOUT)); Assertions.assertEquals(2, calls.get()); } @Test void largeFeedWithContentLengthHeader() { byte[] bytes = new byte[(int) DataSize.kilobytes(10).toBytes()]; Arrays.fill(bytes, (byte) 1); this.mockServerClient.when(HttpRequest.request().withMethod("GET")).respond(HttpResponse.response().withBody(bytes)); IOException e = Assertions.assertThrows(IOException.class, () -> getter.getBinary(this.feedUrl, TIMEOUT)); Assertions.assertEquals("Response size (10000 bytes) exceeds the maximum allowed size (1000 bytes)", e.getMessage()); } }