From 780b7666c54163d6a77b41709ae6f00c1af1d90b Mon Sep 17 00:00:00 2001 From: Athou Date: Mon, 5 Aug 2024 08:25:59 +0200 Subject: [PATCH] add metrics for HttpGetter connection pool --- .../src/pages/admin/MetricsPage.tsx | 23 +++++++++++---- .../com/commafeed/backend/HttpGetter.java | 29 +++++++++++++------ .../com/commafeed/backend/HttpGetterTest.java | 4 ++- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/commafeed-client/src/pages/admin/MetricsPage.tsx b/commafeed-client/src/pages/admin/MetricsPage.tsx index cc5b0f7c..97fa43fc 100644 --- a/commafeed-client/src/pages/admin/MetricsPage.tsx +++ b/commafeed-client/src/pages/admin/MetricsPage.tsx @@ -5,6 +5,7 @@ import { Gauge } from "components/metrics/Gauge" import { Meter } from "components/metrics/Meter" import { MetricAccordionItem } from "components/metrics/MetricAccordionItem" import { Timer } from "components/metrics/Timer" +import { useEffect } from "react" import { useAsync } from "react-async-hook" import { TbChartAreaLine, TbClock } from "react-icons/tb" @@ -18,15 +19,27 @@ const shownMeters: Record = { } const shownGauges: Record = { - "com.commafeed.backend.feed.FeedRefreshEngine.queue.size": "Queue size", - "com.commafeed.backend.feed.FeedRefreshEngine.worker.active": "Feed Worker active", - "com.commafeed.backend.feed.FeedRefreshEngine.updater.active": "Feed Updater active", + "com.commafeed.backend.feed.FeedRefreshEngine.queue.size": "Feed Refresh Engine queue size", + "com.commafeed.backend.feed.FeedRefreshEngine.worker.active": "Feed Refresh Engine active HTTP workers", + "com.commafeed.backend.feed.FeedRefreshEngine.updater.active": "Feed Refresh Engine active database update workers", + "com.commafeed.backend.HttpGetter.pool.max": "HttpGetter max connections", + "com.commafeed.backend.HttpGetter.pool.available": "HttpGetter available connections", + "com.commafeed.backend.HttpGetter.pool.leased": "HttpGetter leased connections", + "com.commafeed.backend.HttpGetter.pool.pending": "HttpGetter waiting for available connections", "com.commafeed.frontend.ws.WebSocketSessions.users": "WebSocket users", "com.commafeed.frontend.ws.WebSocketSessions.sessions": "WebSocket sessions", } export function MetricsPage() { - const query = useAsync(async () => await client.admin.getMetrics(), []) + const query = useAsync(async () => await client.admin.getMetrics(), [], { + // keep previous results available while a new request is pending + setLoading: state => ({ ...state, loading: true }), + }) + + useEffect(() => { + const interval = setInterval(() => query.execute(), 2000) + return () => clearInterval(interval) + }, [query.execute]) if (!query.result) return const { meters, gauges, timers } = query.result.data @@ -53,7 +66,7 @@ export function MetricsPage() { {Object.keys(shownGauges).map(g => ( - {shownGauges[g]}  + {shownGauges[g]}:  ))} diff --git a/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java b/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java index a2d64a95..0f4895bb 100644 --- a/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java +++ b/commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java @@ -15,6 +15,7 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; +import org.apache.hc.client5.http.io.HttpClientConnectionManager; import org.apache.hc.client5.http.protocol.HttpClientContext; import org.apache.hc.client5.http.protocol.RedirectLocations; import org.apache.hc.core5.http.ClassicHttpRequest; @@ -27,6 +28,7 @@ import org.apache.hc.core5.util.TimeValue; import org.apache.hc.core5.util.Timeout; import org.eclipse.jetty.http.HttpStatus; +import com.codahale.metrics.MetricRegistry; import com.commafeed.CommaFeedConfiguration; import com.google.common.collect.Iterables; import com.google.common.io.ByteStreams; @@ -52,11 +54,17 @@ public class HttpGetter { private final DataSize maxResponseSize; @Inject - public HttpGetter(CommaFeedConfiguration config) { + public HttpGetter(CommaFeedConfiguration config, MetricRegistry metrics) { + PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config.getApplicationSettings().getBackgroundThreads()); String userAgent = Optional.ofNullable(config.getApplicationSettings().getUserAgent()) .orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", config.getVersion())); - this.client = newClient(userAgent, config.getApplicationSettings().getBackgroundThreads()); + this.client = newClient(connectionManager, userAgent); this.maxResponseSize = config.getApplicationSettings().getMaxFeedResponseSize(); + + metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "max"), () -> connectionManager.getTotalStats().getMax()); + metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "available"), () -> connectionManager.getTotalStats().getAvailable()); + metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "leased"), () -> connectionManager.getTotalStats().getLeased()); + metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "pending"), () -> connectionManager.getTotalStats().getPending()); } public HttpResult getBinary(String url, int timeout) throws IOException, NotModifiedException { @@ -151,15 +159,10 @@ public class HttpGetter { } } - private static CloseableHttpClient newClient(String userAgent, int poolSize) { + private static PoolingHttpClientConnectionManager newConnectionManager(int poolSize) { SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build(); - List
headers = new ArrayList<>(); - headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en")); - headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache")); - headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache")); - - PoolingHttpClientConnectionManager connectionManager = PoolingHttpClientConnectionManagerBuilder.create() + return PoolingHttpClientConnectionManagerBuilder.create() .setSSLSocketFactory(Apache5SslUtils.toSocketFactory(sslFactory)) .setDefaultConnectionConfig( ConnectionConfig.custom().setConnectTimeout(Timeout.ofSeconds(5)).setTimeToLive(TimeValue.ofSeconds(30)).build()) @@ -167,6 +170,14 @@ public class HttpGetter { .setMaxConnTotal(poolSize) .build(); + } + + private static CloseableHttpClient newClient(HttpClientConnectionManager connectionManager, String userAgent) { + List
headers = new ArrayList<>(); + headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en")); + headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache")); + headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache")); + return HttpClientBuilder.create() .useSystemProperties() .disableAutomaticRetries() diff --git a/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java b/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java index 2a5b568a..06da263d 100644 --- a/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java +++ b/commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java @@ -15,6 +15,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mockito; import org.mockserver.client.MockServerClient; import org.mockserver.junit.jupiter.MockServerExtension; import org.mockserver.model.ConnectionOptions; @@ -23,6 +24,7 @@ import org.mockserver.model.HttpRequest; import org.mockserver.model.HttpResponse; import org.mockserver.model.MediaType; +import com.codahale.metrics.MetricRegistry; import com.commafeed.CommaFeedConfiguration; import com.commafeed.CommaFeedConfiguration.ApplicationSettings; import com.commafeed.backend.HttpGetter.HttpResponseException; @@ -57,7 +59,7 @@ class HttpGetterTest { CommaFeedConfiguration config = new CommaFeedConfiguration(); config.setApplicationSettings(settings); - this.getter = new HttpGetter(config); + this.getter = new HttpGetter(config, Mockito.mock(MetricRegistry.class)); } @ParameterizedTest