From 80c299d915c1e79335831f62242ea40983a504ae Mon Sep 17 00:00:00 2001 From: Mike Zupan Date: Sun, 15 Nov 2026 14:03:30 -0800 Subject: [PATCH] Measure export body completion errors and cancellation --- src/handlers/export.rs | 96 ++++++++++++++++ src/handlers/logs.rs | 15 ++- src/handlers/query.rs | 15 ++- src/main.rs | 38 +++++- src/query_governor.rs | 254 ++++++++++++++++++++++++++++++++++++++--- 5 files changed, 392 insertions(+), 26 deletions(-) diff --git a/src/handlers/export.rs b/src/handlers/export.rs index f876905..91d9e0e 100644 --- a/src/handlers/export.rs +++ b/src/handlers/export.rs @@ -876,6 +876,36 @@ enum StreamKind { Json, } +const EXPORT_ROWS_BUCKETS: &[f64] = &[ + 0.0, + 1.0, + 10.0, + 100.0, + 1_000.0, + 10_000.0, + 50_000.0, + 100_000.0, + 500_000.0, + 1_000_000.0, +]; + +struct SerializedRowsObservation { + metrics: Arc, + format: &'static str, + rows: u64, + outcome: &'static str, +} +impl Drop for SerializedRowsObservation { + fn drop(&mut self) { + self.metrics.observe_histogram_with( + "rush_export_serialized_rows", + &[("format", self.format), ("outcome", self.outcome)], + self.rows as f64, + EXPORT_ROWS_BUCKETS, + ); + } +} + /// State for both CSV and JSON streaming. Only a single decoded row and its /// serialized chunk are live at a time, so peak API memory is independent of /// the requested row count. @@ -888,6 +918,7 @@ struct ExportStreamState { max_bytes: u64, progress: ExportProgress, on_progress: Option, + metrics: SerializedRowsObservation, done: bool, } @@ -903,6 +934,11 @@ impl ExportStreamState { return None; } self.done = true; + self.metrics.outcome = if self.progress.truncated { + "truncated" + } else { + "complete" + }; let suffix = match self.kind { StreamKind::Csv if self.progress.truncated => { b"# truncated: export byte limit reached\n".to_vec() @@ -928,6 +964,7 @@ impl ExportStreamState { } } +#[allow(clippy::too_many_arguments)] fn stream_response( cursor: RowCursor, prelude: Vec, @@ -937,6 +974,7 @@ fn stream_response( max_rows: u64, max_bytes: u64, on_progress: Option, + metrics: Arc, ) -> Response where T: clickhouse::RowOwned + clickhouse::RowRead + Send + 'static, @@ -950,6 +988,15 @@ where max_bytes, progress: ExportProgress::default(), on_progress, + metrics: SerializedRowsObservation { + metrics, + format: match kind { + StreamKind::Csv => "csv", + StreamKind::Json => "json", + }, + rows: 0, + outcome: "cancelled", + }, done: false, }; @@ -960,6 +1007,7 @@ where if let Some(prelude) = state.prelude.take() { if prelude.len() as u64 + STREAM_SUFFIX_RESERVE > state.max_bytes { state.done = true; + state.metrics.outcome = "metadata_limit"; return Some(( Err::(std::io::Error::other( "export metadata exceeds the byte limit", @@ -981,6 +1029,7 @@ where Err(_error) => { tracing::error!(reason = "row_serialization", "export stream failed"); state.done = true; + state.metrics.outcome = "serialization_error"; return Some((Err(std::io::Error::other("export stream failed")), state)); } }; @@ -998,6 +1047,7 @@ where return state.finish_chunk().map(|suffix| (Ok(suffix), state)); } state.progress.rows += 1; + state.metrics.rows = state.progress.rows; state.progress.bytes += chunk.len() as u64; state.notify(); Some((Ok(Bytes::from(chunk)), state)) @@ -1006,6 +1056,7 @@ where Err(_error) => { tracing::error!(reason = "cursor_read", "export stream failed"); state.done = true; + state.metrics.outcome = "cursor_error"; Some((Err(std::io::Error::other("export stream failed")), state)) } } @@ -1043,6 +1094,7 @@ where /// Errors mid-stream terminate the body with an `io::Error`; axum surfaces that as a /// truncated/aborted response. The initial query has already executed by the time the /// first row is pulled, so query-level failures still abort the download cleanly. +#[allow(clippy::too_many_arguments)] pub fn stream_csv_response( cursor: RowCursor, prelude: String, @@ -1051,6 +1103,7 @@ pub fn stream_csv_response( max_rows: u64, max_bytes: u64, on_progress: Option, + metrics: Arc, ) -> Response where T: clickhouse::RowOwned + clickhouse::RowRead + Send + 'static, @@ -1064,6 +1117,7 @@ where max_rows, max_bytes, on_progress, + metrics, ) } @@ -1083,6 +1137,7 @@ pub fn json_query_preamble(query: serde_json::Value) -> Vec { bytes } +#[allow(clippy::too_many_arguments)] pub fn stream_json_response( cursor: RowCursor, prelude: Vec, @@ -1090,6 +1145,7 @@ pub fn stream_json_response( max_rows: u64, max_bytes: u64, on_progress: Option, + metrics: Arc, ) -> Response where T: clickhouse::RowOwned + clickhouse::RowRead + serde::Serialize + Send + 'static, @@ -1103,6 +1159,7 @@ where max_rows, max_bytes, on_progress, + metrics, ) } @@ -1110,6 +1167,45 @@ where mod tests { use super::*; + #[tokio::test] + async fn serialized_row_metrics_follow_actual_cursor_output() { + #[derive(clickhouse::Row, serde::Deserialize, serde::Serialize)] + struct TestRow { + value: u64, + } + let mock = clickhouse::test::Mock::new(); + mock.add(clickhouse::test::handlers::provide([ + TestRow { value: 1 }, + TestRow { value: 2 }, + ])); + let ch = clickhouse::Client::default() + .with_url(mock.url()) + .with_validation(false); + let cursor = ch.query("SELECT value").fetch::().unwrap(); + let metrics = Arc::new(crate::self_metrics::SelfMetrics::new()); + let response = stream_json_response( + cursor, + json_query_preamble(serde_json::json!({})), + "test.json", + 10, + 1024 * 1024, + None, + metrics.clone(), + ); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + assert_eq!( + serde_json::from_slice::(&body).unwrap()["count"], + 2 + ); + assert!( + metrics.render_prometheus().contains( + "rush_export_serialized_rows_sum{format=\"json\",outcome=\"complete\"} 2" + ) + ); + } + #[test] fn async_threshold_applies_to_every_large_effective_request() { assert!(!requires_async(0, sync_max_rows())); diff --git a/src/handlers/logs.rs b/src/handlers/logs.rs index ac0c9bc..21e6811 100644 --- a/src/handlers/logs.rs +++ b/src/handlers/logs.rs @@ -1025,6 +1025,8 @@ pub async fn export_logs( } }; let budget = guard.budget().clone(); + let deadline = tokio::time::Instant::now() + + std::time::Duration::from_secs(budget.request_timeout_secs); let progress = background_state .export_jobs .progress_callback(&background_job_id); @@ -1049,6 +1051,7 @@ pub async fn export_logs( limit, max_bytes, Some(progress), + background_state.self_metrics.clone(), ), export::ExportFormat::Json => export::stream_json_response( cursor, @@ -1057,8 +1060,12 @@ pub async fn export_logs( limit, max_bytes, Some(progress), + background_state.self_metrics.clone(), ), }; + let response = crate::query_governor::retain_artifact_until_body_deadline( + response, guard, deadline, + ); background_state .export_jobs .write_response(&background_job_id, response) @@ -1066,13 +1073,9 @@ pub async fn export_logs( }; let result = crate::query_governor::with_budget( budget.clone(), - tokio::time::timeout( - std::time::Duration::from_secs(budget.request_timeout_secs), - work, - ), + tokio::time::timeout_at(deadline, work), ) .await; - drop(guard); match result { Ok(Ok(())) => {} _ if background_state @@ -1138,6 +1141,7 @@ pub async fn export_logs( limit, max_bytes, None, + state.self_metrics.clone(), )) } export::ExportFormat::Json => { @@ -1168,6 +1172,7 @@ pub async fn export_logs( limit, max_bytes, None, + state.self_metrics.clone(), )) } } diff --git a/src/handlers/query.rs b/src/handlers/query.rs index 5a5dcac..3a62a2b 100644 --- a/src/handlers/query.rs +++ b/src/handlers/query.rs @@ -420,6 +420,8 @@ pub async fn export_query( } }; let budget = guard.budget().clone(); + let deadline = tokio::time::Instant::now() + + std::time::Duration::from_secs(budget.request_timeout_secs); let progress = background_state .export_jobs .progress_callback(&background_job_id); @@ -453,6 +455,7 @@ pub async fn export_query( limit, max_bytes, Some(progress), + background_state.self_metrics.clone(), ), export::ExportFormat::Json => export::stream_json_response( cursor, @@ -461,8 +464,12 @@ pub async fn export_query( limit, max_bytes, Some(progress), + background_state.self_metrics.clone(), ), }; + let response = crate::query_governor::retain_artifact_until_body_deadline( + response, guard, deadline, + ); background_state .export_jobs .write_response(&background_job_id, response) @@ -470,13 +477,9 @@ pub async fn export_query( }; let result = crate::query_governor::with_budget( budget.clone(), - tokio::time::timeout( - std::time::Duration::from_secs(budget.request_timeout_secs), - work, - ), + tokio::time::timeout_at(deadline, work), ) .await; - drop(guard); match result { Ok(Ok(())) => {} _ if background_state @@ -547,6 +550,7 @@ pub async fn export_query( limit, max_bytes, None, + state.self_metrics.clone(), )) } export::ExportFormat::Json => { @@ -577,6 +581,7 @@ pub async fn export_query( limit, max_bytes, None, + state.self_metrics.clone(), )) } } diff --git a/src/main.rs b/src/main.rs index 561c5ff..0525e8e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -434,7 +434,7 @@ async fn http_metrics_middleware( return next.run(req).await; } - let method = req.method().as_str().to_string(); + let method = http_method_label(req.method()).to_string(); let sm = state.self_metrics.clone(); // In-flight gauge: inc on entry, dec on exit (even on panic-free early returns). @@ -468,9 +468,29 @@ async fn http_metrics_middleware( elapsed_ms, ); + sm.observe_histogram( + "rush_http_response_headers_duration_ms", + &[("route", route.as_str()), ("method", method.as_str())], + elapsed_ms, + ); resp } +fn http_method_label(method: &axum::http::Method) -> &'static str { + match method.as_str() { + "GET" => "GET", + "HEAD" => "HEAD", + "POST" => "POST", + "PUT" => "PUT", + "DELETE" => "DELETE", + "CONNECT" => "CONNECT", + "OPTIONS" => "OPTIONS", + "TRACE" => "TRACE", + "PATCH" => "PATCH", + _ => "OTHER", + } +} + /// Open `GET /metrics` handler: renders the self-metrics registry as Prometheus text /// exposition (version 0.0.4). No auth — same posture as `/healthz`. async fn metrics_handler(State(state): State) -> Response { @@ -1452,6 +1472,11 @@ where .await; match result { Ok((response, outcome)) => { + let outcome = if class == WorkloadClass::Export && outcome == "completed" { + "response_ready" + } else { + outcome + }; metrics.inc_counter( "rush_query_requests_total", &[("workload", label), ("outcome", outcome)], @@ -4474,6 +4499,17 @@ mod tenant_auth_tests { use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; + #[test] + fn arbitrary_http_methods_do_not_create_unbounded_metric_labels() { + assert_eq!(super::http_method_label(&axum::http::Method::GET), "GET"); + assert_eq!( + super::http_method_label( + &axum::http::Method::from_bytes(b"CUSTOM-TENANT-SECRET").unwrap() + ), + "OTHER" + ); + } + #[test] fn startup_validates_api_key_secret_before_running_migrations() { let source = include_str!("main.rs"); diff --git a/src/query_governor.rs b/src/query_governor.rs index 0a1207d..8438b3a 100644 --- a/src/query_governor.rs +++ b/src/query_governor.rs @@ -582,6 +582,73 @@ pub fn retain_admission_until_body_end( retain_admission_until_body_deadline(response, guard, deadline) } +const EXPORT_BODY_LATENCY_BUCKETS_MS: &[f64] = &[ + 10.0, 50.0, 100.0, 500.0, 1_000.0, 5_000.0, 10_000.0, 30_000.0, 60_000.0, 120_000.0, 300_000.0, + 600_000.0, +]; +const EXPORT_BODY_BYTES_BUCKETS: &[f64] = &[ + 0.0, + 1_024.0, + 16_384.0, + 262_144.0, + 1_048_576.0, + 4_194_304.0, + 16_777_216.0, + 67_108_864.0, + 268_435_456.0, + 1_073_741_824.0, + 4_294_967_296.0, +]; + +struct ExportBodyObservation { + metrics: Arc, + kind: &'static str, + started: std::time::Instant, + bytes: u64, + recorded: bool, +} + +impl ExportBodyObservation { + fn new(metrics: Arc, kind: &'static str) -> Self { + metrics.add_gauge("rush_export_streams_inflight", &[("kind", kind)], 1.0); + Self { + metrics, + kind, + started: std::time::Instant::now(), + bytes: 0, + recorded: false, + } + } + fn finish(&mut self, outcome: &'static str) { + if self.recorded { + return; + } + self.recorded = true; + self.metrics + .add_gauge("rush_export_streams_inflight", &[("kind", self.kind)], -1.0); + let labels = [("kind", self.kind), ("outcome", outcome)]; + self.metrics + .inc_counter("rush_export_streams_total", &labels, 1); + self.metrics.observe_histogram_with( + "rush_export_stream_duration_ms", + &labels, + self.started.elapsed().as_secs_f64() * 1000.0, + EXPORT_BODY_LATENCY_BUCKETS_MS, + ); + self.metrics.observe_histogram_with( + "rush_export_stream_bytes", + &labels, + self.bytes as f64, + EXPORT_BODY_BYTES_BUCKETS, + ); + } +} +impl Drop for ExportBodyObservation { + fn drop(&mut self) { + self.finish("cancelled"); + } +} + /// Use the request's original deadline so body upload and handler execution do /// not grant a fresh timeout when an export starts streaming. pub fn retain_admission_until_body_deadline( @@ -589,58 +656,92 @@ pub fn retain_admission_until_body_deadline( guard: QueryAdmissionGuard, deadline: tokio::time::Instant, ) -> axum::response::Response { + observe_export_body(response, guard, deadline, "response") +} + +/// Background artifact generation has the same lifecycle but a separate kind, +/// so its bytes are not confused with a later HTTP artifact download. +pub fn retain_artifact_until_body_deadline( + response: axum::response::Response, + guard: QueryAdmissionGuard, + deadline: tokio::time::Instant, +) -> axum::response::Response { + observe_export_body(response, guard, deadline, "artifact") +} + +fn observe_export_body( + response: axum::response::Response, + guard: QueryAdmissionGuard, + deadline: tokio::time::Instant, + kind: &'static str, +) -> axum::response::Response { + let observation = ExportBodyObservation::new(guard.metrics.clone(), kind); let (parts, body) = response.into_parts(); let state = ( - Box::pin(body.into_data_stream()), + Some(Box::pin(body.into_data_stream())), Some(guard), deadline, false, + observation, ); - let stream = - futures_util::stream::unfold(state, |(mut body, mut guard, deadline, done)| async move { + let stream = futures_util::stream::unfold( + state, + |(mut body, mut guard, deadline, done, mut observation)| async move { if done { return None; } - // timeout_at polls the future first. Explicitly enforce an expired - // deadline even if the next chunk is immediately ready. if tokio::time::Instant::now() >= deadline { + observation.finish("timeout"); + body.take(); guard.take(); return Some(( Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "export response exceeded its workload timeout", )), - (body, guard, deadline, true), + (body, guard, deadline, true, observation), )); } - match tokio::time::timeout_at(deadline, body.next()).await { - Ok(Some(Ok(bytes))) => Some(( - Ok::(bytes), - (body, guard, deadline, false), - )), + let next = + tokio::time::timeout_at(deadline, body.as_mut().expect("active body").next()).await; + match next { + Ok(Some(Ok(bytes))) => { + observation.bytes = observation.bytes.saturating_add(bytes.len() as u64); + Some(( + Ok::(bytes), + (body, guard, deadline, false, observation), + )) + } Ok(Some(Err(error))) => { + observation.finish("body_error"); + body.take(); guard.take(); Some(( Err(std::io::Error::other(error)), - (body, guard, deadline, true), + (body, guard, deadline, true, observation), )) } Ok(None) => { + observation.finish("complete"); + body.take(); guard.take(); None } Err(_) => { + observation.finish("timeout"); + body.take(); guard.take(); Some(( Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "export response exceeded its workload timeout", )), - (body, guard, deadline, true), + (body, guard, deadline, true, observation), )) } } - }); + }, + ); axum::response::Response::from_parts(parts, axum::body::Body::from_stream(stream)) } @@ -916,13 +1017,131 @@ mod tests { assert!(dropped.load(Ordering::SeqCst)); } + #[tokio::test] + async fn stream_metrics_distinguish_eof_error_timeout_and_artifact_generation() { + let metrics = Arc::new(SelfMetrics::new()); + let governor = QueryGovernor::new(QueryGovernorConfig::default(), metrics.clone()).unwrap(); + let guard = governor + .admit(WorkloadClass::Export, "tenant-secret") + .await + .unwrap(); + let response = retain_admission_until_body_end( + axum::response::Response::new(axum::body::Body::from("hello")), + guard, + Duration::from_secs(1), + ); + assert!( + metrics + .render_prometheus() + .contains("rush_export_streams_inflight{kind=\"response\"} 1") + ); + assert!( + !metrics + .render_prometheus() + .contains("rush_export_streams_total") + ); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap() + .len(), + 5 + ); + let guard = governor + .admit(WorkloadClass::Export, "tenant-secret") + .await + .unwrap(); + let body = axum::body::Body::from_stream(futures_util::stream::once(async { + Err::(std::io::Error::other("secret error")) + })); + assert!( + axum::body::to_bytes( + retain_admission_until_body_end( + axum::response::Response::new(body), + guard, + Duration::from_secs(1) + ) + .into_body(), + 1024 + ) + .await + .is_err() + ); + let guard = governor + .admit(WorkloadClass::Export, "tenant-secret") + .await + .unwrap(); + let response = retain_artifact_until_body_deadline( + axum::response::Response::new(axum::body::Body::from("artifact")), + guard, + tokio::time::Instant::now() + Duration::from_secs(1), + ); + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(); + let text = metrics.render_prometheus(); + assert!( + text.contains("rush_export_streams_total{kind=\"response\",outcome=\"complete\"} 1") + ); + assert!( + text.contains("rush_export_streams_total{kind=\"response\",outcome=\"body_error\"} 1") + ); + assert!( + text.contains("rush_export_streams_total{kind=\"artifact\",outcome=\"complete\"} 1") + ); + assert!( + text.contains("rush_export_stream_bytes_sum{kind=\"response\",outcome=\"complete\"} 5") + ); + assert!(!text.contains("secret")); + } + + #[tokio::test] + async fn timed_out_stream_drops_cursor_even_if_consumer_keeps_failed_body() { + struct DropWatch(Arc); + impl Drop for DropWatch { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } + } + let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let watch = DropWatch(dropped.clone()); + let stream = futures_util::stream::pending::>().map( + move |chunk| { + let _ = &watch; + chunk + }, + ); + let metrics = Arc::new(SelfMetrics::new()); + let governor = QueryGovernor::new(QueryGovernorConfig::default(), metrics.clone()).unwrap(); + let guard = governor + .admit(WorkloadClass::Export, "tenant-a") + .await + .unwrap(); + let response = retain_admission_until_body_end( + axum::response::Response::new(axum::body::Body::from_stream(stream)), + guard, + Duration::from_millis(5), + ); + let mut body = response.into_body().into_data_stream(); + assert!(body.next().await.unwrap().is_err()); + assert!(dropped.load(Ordering::SeqCst)); + let text = metrics.render_prometheus(); + assert!(text.contains("rush_export_streams_inflight{kind=\"response\"} 0")); + assert!( + text.contains("rush_export_streams_total{kind=\"response\",outcome=\"timeout\"} 1") + ); + drop(body); + assert_eq!(text, metrics.render_prometheus()); + } + #[tokio::test] async fn streaming_response_holds_permit_until_body_is_dropped() { let mut config = QueryGovernorConfig::default(); config.export.global_concurrency = 1; config.export.per_tenant_concurrency = 1; config.export.queue_timeout_ms = 0; - let governor = QueryGovernor::new(config, Arc::new(SelfMetrics::new())).unwrap(); + let metrics = Arc::new(SelfMetrics::new()); + let governor = QueryGovernor::new(config, metrics.clone()).unwrap(); let guard = governor .admit(WorkloadClass::Export, "tenant-a") .await @@ -941,6 +1160,11 @@ mod tests { Err(AdmissionError::TenantBusy { .. }) )); drop(response); + let text = metrics.render_prometheus(); + assert!( + text.contains("rush_export_streams_total{kind=\"response\",outcome=\"cancelled\"} 1") + ); + assert!(text.contains("rush_export_streams_inflight{kind=\"response\"} 0")); assert!( governor .admit(WorkloadClass::Export, "tenant-a")