diff --git a/pom.xml b/pom.xml index 877a51c..153f992 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 fi.hsl transitdata-common - 3.0.0 + 3.1.0 jar Common utilities for Transitdata projects @@ -14,6 +14,7 @@ 2.43.0 1.23.0 2.20.1 + 0.16.0 @@ -161,7 +162,16 @@ 3.26.0 test - + + io.prometheus + simpleclient + ${prometheus.version} + + + io.prometheus + simpleclient_httpserver + ${prometheus.version} + diff --git a/src/main/java/fi/hsl/common/metrics/MetricsServer.java b/src/main/java/fi/hsl/common/metrics/MetricsServer.java new file mode 100644 index 0000000..b71ec32 --- /dev/null +++ b/src/main/java/fi/hsl/common/metrics/MetricsServer.java @@ -0,0 +1,27 @@ +package fi.hsl.common.metrics; + +import io.prometheus.client.exporter.HTTPServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class MetricsServer implements AutoCloseable { + private static final Logger log = LoggerFactory.getLogger(MetricsServer.class); + + public final int port; + private final HTTPServer httpServer; + + public MetricsServer(final int port) throws IOException { + this.port = port; + log.info("Starting MetricsServer on port {}, endpoint /metrics", port); + this.httpServer = new HTTPServer.Builder().withPort(port).build(); + } + + @Override + public void close() { + if (httpServer != null) { + httpServer.close(); + } + } +} diff --git a/src/main/java/fi/hsl/common/metrics/PipelineMetrics.java b/src/main/java/fi/hsl/common/metrics/PipelineMetrics.java new file mode 100644 index 0000000..989bace --- /dev/null +++ b/src/main/java/fi/hsl/common/metrics/PipelineMetrics.java @@ -0,0 +1,28 @@ +package fi.hsl.common.metrics; + +import io.prometheus.client.Histogram; + +public final class PipelineMetrics { + + private static final double[] LATENCY_BUCKETS = {0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120}; + + private static final Histogram E2E_LATENCY = Histogram.build().name("transitdata_e2e_latency_seconds") + .help("Internal processing latency from ingest to produced output, per pipeline.").labelNames("pipeline") + .buckets(LATENCY_BUCKETS).register(); + + private static final Histogram PUBLISH_DURATION = Histogram.build() + .name("transitdata_feed_publish_duration_seconds") + .help("Time to build and write a full feed dump, per pipeline.").labelNames("pipeline") + .buckets(LATENCY_BUCKETS).register(); + + private PipelineMetrics() { + } + + public static void observeLatency(final String pipeline, final double seconds) { + E2E_LATENCY.labels(pipeline).observe(seconds); + } + + public static void observePublishDuration(final String pipeline, final double seconds) { + PUBLISH_DURATION.labels(pipeline).observe(seconds); + } +} diff --git a/src/main/java/fi/hsl/common/pulsar/PulsarApplication.java b/src/main/java/fi/hsl/common/pulsar/PulsarApplication.java index 2888766..b6a394c 100644 --- a/src/main/java/fi/hsl/common/pulsar/PulsarApplication.java +++ b/src/main/java/fi/hsl/common/pulsar/PulsarApplication.java @@ -2,6 +2,7 @@ import com.typesafe.config.Config; import fi.hsl.common.health.HealthServer; +import fi.hsl.common.metrics.MetricsServer; import fi.hsl.common.redis.RedisClusterProperties; import fi.hsl.common.redis.RedisStore; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -46,6 +47,7 @@ public class PulsarApplication implements AutoCloseable { PulsarAdmin admin; RedisStore redisStore; HealthServer healthServer; + MetricsServer metricsServer; PulsarApplication() { } @@ -120,6 +122,10 @@ public PulsarApplicationContext initialize(@NotNull Config config) throws Except } } + if (config.hasPath("metrics.enabled") && config.getBoolean("metrics.enabled")) { + metricsServer = new MetricsServer(config.getInt("metrics.port")); + } + return createContext(config, client, consumer, producers, redisStore, admin, healthServer); } @@ -322,5 +328,7 @@ public void close() { redisStore.close(); if (healthServer != null) healthServer.close(); + if (metricsServer != null) + metricsServer.close(); } } diff --git a/src/main/resources/common.conf b/src/main/resources/common.conf index bd924ac..910e9cf 100644 --- a/src/main/resources/common.conf +++ b/src/main/resources/common.conf @@ -63,6 +63,13 @@ health { endpoint=${?HEALTH_ENDPOINT} } +metrics { + enabled=false + enabled=${?METRICS_ENABLED} + port=8080 + port=${?METRICS_PORT} +} + gtfsrt { serviceDayStartTime="04:30:00" serviceDayStartTime=${?GTFSRT_SERVICE_DAY_START_TIME}