From a90dd566165fa7497d55e2bc2db831103e9d51ec Mon Sep 17 00:00:00 2001
From: Rex Raphael <rex.raphael@outlook.com>
Date: Sun, 18 Oct 2026 08:04:01 -0500
Subject: [PATCH] test(extension): scrape until the job metrics settle

The v1.9.0 release run failed on TestExtensionRecordsThroughForge. The
test scraped once, right after it saw the "process job" span end, but the
worker drops dispatch_jobs_active back to 0 just after that span ends, so
a quick scrape could still read 1. The test now scrapes until every
expected series is there (up to 5s) and then checks. It passed 200 runs
under -race, and 100 more with GOMAXPROCS=2 on a loaded machine.
---
 extension/telemetry_test.go | 31 ++++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/extension/telemetry_test.go b/extension/telemetry_test.go
index 93c8bb2..91c725c 100644
--- a/extension/telemetry_test.go
+++ b/extension/telemetry_test.go
@@ -34,6 +34,27 @@ func scrape(t *testing.T, app forge.App) string {
 	return string(body)
 }
 
+// scrapeUntil scrapes app's metrics until the body holds every line in
+// wants or the timeout passes, and returns the last body it read.
+func scrapeUntil(t *testing.T, app forge.App, wants []string, timeout time.Duration) string {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for {
+		body := scrape(t, app)
+		missing := false
+		for _, want := range wants {
+			if !strings.Contains(body, want) {
+				missing = true
+				break
+			}
+		}
+		if !missing || time.Now().After(deadline) {
+			return body
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+}
+
 func TestExtensionRecordsThroughForge(t *testing.T) {
 	prevTP, prevProp := otel.GetTracerProvider(), otel.GetTextMapPropagator()
 	t.Cleanup(func() {
@@ -79,8 +100,7 @@ func TestExtensionRecordsThroughForge(t *testing.T) {
 		}
 	}
 
-	body := scrape(t, app)
-	for _, want := range []string{
+	wants := []string{
 		`forge_dispatch_jobs_enqueued_total{queue="default"} 1`,
 		`forge_dispatch_job_attempt_duration_seconds_count{outcome="completed",queue="default"} 1`,
 		`forge_dispatch_job_queue_wait_seconds_count{queue="default"} 1`,
@@ -89,7 +109,12 @@ func TestExtensionRecordsThroughForge(t *testing.T) {
 		`forge_dispatch_cron_leader`,
 		// The legacy counters keep their names.
 		`forge_dispatch_job_enqueued 1`,
-	} {
+	}
+	// The worker drops jobs_active back to 0 just after the "process job"
+	// span ends, so a scrape taken the moment the span is seen can still
+	// read 1. Scrape until every series has settled, then check.
+	body := scrapeUntil(t, app, wants, 5*time.Second)
+	for _, want := range wants {
 		if !strings.Contains(body, want) {
 			t.Errorf("scrape is missing %s", want)
 		}