Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-03-21 21:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

add metrics for HttpGetter connection pool

Browse Source
This commit is contained in:
Athou
2024-08-05 08:25:59 +02:00
parent 70b4534e14
commit 780b7666c5
3 changed files with 41 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
commafeed-client/src/pages/admin/MetricsPage.tsx
View File

@@ -5,6 +5,7 @@ import { Gauge } from "components/metrics/Gauge"
import { Meter } from "components/metrics/Meter"
import { MetricAccordionItem } from "components/metrics/MetricAccordionItem"
import { Timer } from "components/metrics/Timer"
import { useEffect } from "react"
import { useAsync } from "react-async-hook"
import { TbChartAreaLine, TbClock } from "react-icons/tb"
@@ -18,15 +19,27 @@ const shownMeters: Record<string, string> = {
}
const shownGauges: Record<string, string> = {
"com.commafeed.backend.feed.FeedRefreshEngine.queue.size": "Queue size",
"com.commafeed.backend.feed.FeedRefreshEngine.worker.active": "Feed Worker active",
"com.commafeed.backend.feed.FeedRefreshEngine.updater.active": "Feed Updater active",
"com.commafeed.backend.feed.FeedRefreshEngine.queue.size": "Feed Refresh Engine queue size",
"com.commafeed.backend.feed.FeedRefreshEngine.worker.active": "Feed Refresh Engine active HTTP workers",
"com.commafeed.backend.feed.FeedRefreshEngine.updater.active": "Feed Refresh Engine active database update workers",
"com.commafeed.backend.HttpGetter.pool.max": "HttpGetter max connections",
"com.commafeed.backend.HttpGetter.pool.available": "HttpGetter available connections",
"com.commafeed.backend.HttpGetter.pool.leased": "HttpGetter leased connections",
"com.commafeed.backend.HttpGetter.pool.pending": "HttpGetter waiting for available connections",
"com.commafeed.frontend.ws.WebSocketSessions.users": "WebSocket users",
"com.commafeed.frontend.ws.WebSocketSessions.sessions": "WebSocket sessions",
}
export function MetricsPage() {
const query = useAsync(async () => await client.admin.getMetrics(), [])
const query = useAsync(async () => await client.admin.getMetrics(), [], {
// keep previous results available while a new request is pending
setLoading: state => ({ ...state, loading: true }),
})
useEffect(() => {
const interval = setInterval(() => query.execute(), 2000)
return () => clearInterval(interval)
}, [query.execute])
if (!query.result) return <Loader />
const { meters, gauges, timers } = query.result.data
@@ -53,7 +66,7 @@ export function MetricsPage() {
<Box pt="xs">
{Object.keys(shownGauges).map(g => (
<Box key={g}>
<span>{shownGauges[g]}&nbsp;</span>
<span>{shownGauges[g]}:&nbsp;</span>
<Gauge gauge={gauges[g]} />
</Box>
))}

29
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -15,6 +15,7 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.client5.http.protocol.RedirectLocations;
import org.apache.hc.core5.http.ClassicHttpRequest;
@@ -27,6 +28,7 @@ import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.eclipse.jetty.http.HttpStatus;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
@@ -52,11 +54,17 @@ public class HttpGetter {
private final DataSize maxResponseSize;
@Inject
public HttpGetter(CommaFeedConfiguration config) {
public HttpGetter(CommaFeedConfiguration config, MetricRegistry metrics) {
PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config.getApplicationSettings().getBackgroundThreads());
String userAgent = Optional.ofNullable(config.getApplicationSettings().getUserAgent())
.orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", config.getVersion()));
this.client = newClient(userAgent, config.getApplicationSettings().getBackgroundThreads());
this.client = newClient(connectionManager, userAgent);
this.maxResponseSize = config.getApplicationSettings().getMaxFeedResponseSize();
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "max"), () -> connectionManager.getTotalStats().getMax());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "available"), () -> connectionManager.getTotalStats().getAvailable());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "leased"), () -> connectionManager.getTotalStats().getLeased());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "pending"), () -> connectionManager.getTotalStats().getPending());
}
public HttpResult getBinary(String url, int timeout) throws IOException, NotModifiedException {
@@ -151,15 +159,10 @@ public class HttpGetter {
}
}
private static CloseableHttpClient newClient(String userAgent, int poolSize) {
private static PoolingHttpClientConnectionManager newConnectionManager(int poolSize) {
SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
PoolingHttpClientConnectionManager connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
return PoolingHttpClientConnectionManagerBuilder.create()
.setSSLSocketFactory(Apache5SslUtils.toSocketFactory(sslFactory))
.setDefaultConnectionConfig(
ConnectionConfig.custom().setConnectTimeout(Timeout.ofSeconds(5)).setTimeToLive(TimeValue.ofSeconds(30)).build())
@@ -167,6 +170,14 @@ public class HttpGetter {
.setMaxConnTotal(poolSize)
.build();
}
private static CloseableHttpClient newClient(HttpClientConnectionManager connectionManager, String userAgent) {
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
return HttpClientBuilder.create()
.useSystemProperties()
.disableAutomaticRetries()

4
commafeed-server/src/test/java/com/commafeed/backend/HttpGetterTest.java
View File

@@ -15,6 +15,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.Mockito;
import org.mockserver.client.MockServerClient;
import org.mockserver.junit.jupiter.MockServerExtension;
import org.mockserver.model.ConnectionOptions;
@@ -23,6 +24,7 @@ import org.mockserver.model.HttpRequest;
import org.mockserver.model.HttpResponse;
import org.mockserver.model.MediaType;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.ApplicationSettings;
import com.commafeed.backend.HttpGetter.HttpResponseException;
@@ -57,7 +59,7 @@ class HttpGetterTest {
CommaFeedConfiguration config = new CommaFeedConfiguration();
config.setApplicationSettings(settings);
this.getter = new HttpGetter(config);
this.getter = new HttpGetter(config, Mockito.mock(MetricRegistry.class));
}
@ParameterizedTest