From 71e7b60ae5ce9b577d5642f179f759932df6caed Mon Sep 17 00:00:00 2001 From: Mike Zupan Date: Sun, 15 Nov 2026 13:50:14 -0800 Subject: [PATCH 1/5] Report stats persistence failures and probe validity --- src/stats_engine.rs | 292 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 226 insertions(+), 66 deletions(-) diff --git a/src/stats_engine.rs b/src/stats_engine.rs index 359469a..6e245af 100644 --- a/src/stats_engine.rs +++ b/src/stats_engine.rs @@ -5,7 +5,7 @@ use futures_util::stream::{self, StreamExt}; use std::sync::Arc; use std::time::Instant; -#[derive(clickhouse::Row, serde::Deserialize)] +#[derive(clickhouse::Row, serde::Deserialize, serde::Serialize)] struct CountRow { count: u64, } @@ -15,7 +15,7 @@ struct BytesRow { total: u64, } -#[derive(clickhouse::Row, serde::Deserialize)] +#[derive(clickhouse::Row, serde::Deserialize, serde::Serialize)] struct F64Row { v: f64, } @@ -101,26 +101,34 @@ async fn collect_and_write( disk_local_total_bytes, buf_oldest, ) = tokio::join!( - query_count(ch, &q_spans), + query_count(ch, &q_spans, self_metrics, "spans"), query_bytes( ch, - "SELECT sum(bytes_on_disk) as total FROM system.parts WHERE database = 'observability' AND table = 'spans' AND active" + "SELECT sum(bytes_on_disk) as total FROM system.parts WHERE database = 'observability' AND table = 'spans' AND active", + self_metrics, + "span_bytes" ), - query_count(ch, &q_logs), - query_count(ch, &q_gauge), - query_count(ch, &q_sum), - query_count(ch, &q_hist), + query_count(ch, &q_logs, self_metrics, "logs"), + query_count(ch, &q_gauge, self_metrics, "metric_gauge"), + query_count(ch, &q_sum, self_metrics, "metric_sum"), + query_count(ch, &q_hist, self_metrics, "metric_histogram"), query_count( ch, - "SELECT uniq(MetricName, Attributes) as count FROM metrics_gauge WHERE TimeUnix >= now() - INTERVAL 1 HOUR" + "SELECT uniq(MetricName, Attributes) as count FROM metrics_gauge WHERE TimeUnix >= now() - INTERVAL 1 HOUR", + self_metrics, + "unique_series" ), query_bytes( ch, - "SELECT sum(bytes_on_disk) as total FROM system.parts WHERE database = 'observability' AND active" + "SELECT sum(bytes_on_disk) as total FROM system.parts WHERE database = 'observability' AND active", + self_metrics, + "storage_bytes" ), query_count( ch, - "SELECT sum(rows) as count FROM system.parts WHERE database = 'observability' AND active" + "SELECT sum(rows) as count FROM system.parts WHERE database = 'observability' AND active", + self_metrics, + "storage_rows" ), // Tiered storage breakdown: data bytes on local disk vs object store. // Classified by joining each part's disk to system.disks.type, matching @@ -129,71 +137,105 @@ async fn collect_and_write( ch, "SELECT sum(p.bytes_on_disk) as total FROM system.parts p \ LEFT JOIN system.disks d ON p.disk_name = d.name \ - WHERE p.database = 'observability' AND p.active AND d.type = 'Local'" + WHERE p.database = 'observability' AND p.active AND d.type = 'Local'", + self_metrics, + "storage_local_bytes" ), query_bytes( ch, "SELECT sum(p.bytes_on_disk) as total FROM system.parts p \ LEFT JOIN system.disks d ON p.disk_name = d.name \ - WHERE p.database = 'observability' AND p.active AND d.type != 'Local'" + WHERE p.database = 'observability' AND p.active AND d.type != 'Local'", + self_metrics, + "storage_object_store_bytes" ), // Local disk capacity (headroom) from system.disks. query_bytes( ch, - "SELECT sum(free_space) as total FROM system.disks WHERE type = 'Local'" + "SELECT sum(free_space) as total FROM system.disks WHERE type = 'Local'", + self_metrics, + "disk_local_free_bytes" ), query_bytes( ch, - "SELECT sum(total_space) as total FROM system.disks WHERE type = 'Local'" + "SELECT sum(total_space) as total FROM system.disks WHERE type = 'Local'", + self_metrics, + "disk_local_total_bytes" ), // Ingest buffer (durable spool) replay lag. buffer.oldest_age_secs(), ); - let metric_total = metric_gauge + metric_sum + metric_hist; + let metric_total = metric_gauge + .zip(metric_sum) + .zip(metric_hist) + .map(|((a, b), c)| a + b + c); + let mut healthy = [ + span_total, + span_bytes, + log_total, + metric_gauge, + metric_sum, + metric_hist, + unique_series, + storage_bytes, + storage_rows, + storage_local_bytes, + storage_object_store_bytes, + disk_local_free_bytes, + disk_local_total_bytes, + ] + .iter() + .all(Option::is_some); let buf_oldest = buf_oldest.unwrap_or(0); // ── Write all metrics ── let metrics: Vec<(&str, f64)> = vec![ ( "rush_stats_ingest_buffer_pending_bytes", - buffer.total_bytes() as f64, + Some(buffer.total_bytes() as f64), ), ( "rush_stats_ingest_buffer_pending_count", - buffer.segment_count() as f64, + Some(buffer.segment_count() as f64), ), ( "rush_stats_ingest_buffer_oldest_age_secs", - buf_oldest as f64, + Some(buf_oldest as f64), ), // Cumulative counter — drain rate = rate(rush_stats_ingest_buffer_committed_total). ( "rush_stats_ingest_buffer_committed_total", - buffer.committed_total() as f64, + Some(buffer.committed_total() as f64), ), - ("rush_stats_span_events_total", span_total as f64), - ("rush_stats_span_events_bytes", span_bytes as f64), - ("rush_stats_logs_total", log_total as f64), - ("rush_stats_metrics_total", metric_total as f64), - ("rush_stats_unique_series", unique_series as f64), - ("rush_stats_storage_bytes", storage_bytes as f64), - ("rush_stats_storage_rows", storage_rows as f64), + ("rush_stats_span_events_total", span_total.map(|v| v as f64)), + ("rush_stats_span_events_bytes", span_bytes.map(|v| v as f64)), + ("rush_stats_logs_total", log_total.map(|v| v as f64)), + ("rush_stats_metrics_total", metric_total.map(|v| v as f64)), + ("rush_stats_unique_series", unique_series.map(|v| v as f64)), + ("rush_stats_storage_bytes", storage_bytes.map(|v| v as f64)), + ("rush_stats_storage_rows", storage_rows.map(|v| v as f64)), // Tiered storage: where the data physically lives. - ("rush_stats_storage_local_bytes", storage_local_bytes as f64), + ( + "rush_stats_storage_local_bytes", + storage_local_bytes.map(|v| v as f64), + ), ( "rush_stats_storage_object_store_bytes", - storage_object_store_bytes as f64, + storage_object_store_bytes.map(|v| v as f64), ), // Local disk capacity, for headroom / move-pressure monitoring. ( "rush_stats_disk_local_free_bytes", - disk_local_free_bytes as f64, + disk_local_free_bytes.map(|v| v as f64), ), ( "rush_stats_disk_local_total_bytes", - disk_local_total_bytes as f64, + disk_local_total_bytes.map(|v| v as f64), ), - ]; + ] + .into_iter() + .filter_map(|(name, value)| value.map(|value| (name, value))) + .collect(); let instance_attrs = labels_to_ch_map(&[("instance", instance_id.to_string())]); let values: Vec = metrics @@ -216,9 +258,7 @@ async fn collect_and_write( values.join(", ") ); - if let Err(e) = ch.query(&sql).execute().await { - tracing::warn!("stats engine: metric write failed: {e}"); - } + healthy &= write_self_metrics(ch, self_metrics, &sql, "stats", "metrics_gauge").await; tracing::debug!("stats engine: wrote {} metrics", metrics.len()); @@ -243,18 +283,18 @@ async fn collect_and_write( // Storage capacity gauges are process-global and intentionally have no // tenant labels. They power the admin capacity view and keep disk headroom // visible even when no recent stats snapshot has been written yet. - self_metrics.set_gauge( - "rush_stats_disk_local_free_bytes", - &[], - disk_local_free_bytes as f64, - ); - self_metrics.set_gauge( - "rush_stats_disk_local_total_bytes", - &[], - disk_local_total_bytes as f64, - ); - self_metrics.set_gauge("rush_stats_storage_bytes", &[], storage_bytes as f64); - self_metrics.set_gauge("rush_stats_storage_rows", &[], storage_rows as f64); + if let Some(value) = disk_local_free_bytes { + self_metrics.set_gauge("rush_stats_disk_local_free_bytes", &[], value as f64); + } + if let Some(value) = disk_local_total_bytes { + self_metrics.set_gauge("rush_stats_disk_local_total_bytes", &[], value as f64); + } + if let Some(value) = storage_bytes { + self_metrics.set_gauge("rush_stats_storage_bytes", &[], value as f64); + } + if let Some(value) = storage_rows { + self_metrics.set_gauge("rush_stats_storage_rows", &[], value as f64); + } // ── ClickHouse health gauges (group D) into both SelfMetrics and metrics_gauge ── collect_ch_health(ch, self_metrics).await; @@ -262,8 +302,11 @@ async fn collect_and_write( // ── Self-ingest: write the SelfMetrics snapshot into our own metrics tables so the // same rush_http_*/rush_ingest_*/rush_engine_*/rush_ch_* series are queryable via // the in-product PromQL API (/prom/api/v1/*). ── - self_ingest_snapshot(ch, self_metrics, now_nanos, instance_id).await; - + healthy &= self_ingest_snapshot(ch, self_metrics, now_nanos, instance_id).await; + anyhow::ensure!( + healthy, + "stats collection or self-metrics persistence was degraded" + ); Ok(()) } @@ -501,6 +544,19 @@ async fn run_ch_probe( } fn record_ch_probe(self_metrics: &SelfMetrics, probe: &'static str, start: Instant, ok: bool) { + let labels = [("probe", probe)]; + self_metrics.set_gauge( + "rush_ch_health_probe_valid", + &labels, + if ok { 1.0 } else { 0.0 }, + ); + if ok { + self_metrics.set_gauge( + "rush_ch_health_probe_last_success_timestamp_seconds", + &labels, + chrono::Utc::now().timestamp() as f64, + ); + } let outcome = if ok { "ok" } else { "error" }; self_metrics.inc_counter( "rush_ch_health_probes_total", @@ -524,16 +580,17 @@ fn spool_utilization_ratio(bytes: u64, max_bytes: u64) -> f64 { /// Insert the SelfMetrics snapshot into `metrics_gauge` / `metrics_sum` (by kind). Labels /// become the ClickHouse `Attributes` map. Reuses the same raw-SQL insert path the -/// rush_stats_* gauges use. Best-effort: a failed insert is logged, not fatal to the tick. +/// rush_stats_* gauges use. Both table writes are attempted; a failed insert degrades the tick. async fn self_ingest_snapshot( ch: &Client, self_metrics: &SelfMetrics, now_nanos: i64, instance_id: &str, -) { +) -> bool { + let mut healthy = true; let points = self_metrics.snapshot_series(); if points.is_empty() { - return; + return true; } let mut gauge_rows: Vec = Vec::new(); @@ -569,9 +626,7 @@ async fn self_ingest_snapshot( Exemplars.SpanId, Exemplars.TraceId) VALUES {}", gauge_rows.join(", ") ); - if let Err(e) = ch.query(&sql).execute().await { - tracing::warn!("stats engine: self-metrics gauge ingest failed: {e}"); - } + healthy &= write_self_metrics(ch, self_metrics, &sql, "snapshot", "metrics_gauge").await; } if !sum_rows.is_empty() { @@ -584,10 +639,9 @@ async fn self_ingest_snapshot( Exemplars.SpanId, Exemplars.TraceId, AggregationTemporality, IsMonotonic) VALUES {}", sum_rows.join(", ") ); - if let Err(e) = ch.query(&sql).execute().await { - tracing::warn!("stats engine: self-metrics sum ingest failed: {e}"); - } + healthy &= write_self_metrics(ch, self_metrics, &sql, "snapshot", "metrics_sum").await; } + healthy } /// Add the deployment's stable identity to every self-ingested series. The in-process @@ -640,26 +694,132 @@ fn escape_sql(s: &str) -> String { crate::query_builder::escape_string_literal(s) } -async fn query_count(ch: &Client, sql: &str) -> u64 { - ch.query(sql) +async fn write_self_metrics( + ch: &Client, + metrics: &SelfMetrics, + sql: &str, + stage: &'static str, + table: &'static str, +) -> bool { + let started = Instant::now(); + let result = ch.query(sql).execute().await; + let outcome = if result.is_ok() { "ok" } else { "error" }; + let labels = [("stage", stage), ("table", table), ("outcome", outcome)]; + metrics.inc_counter("rush_self_metrics_writes_total", &labels, 1); + metrics.observe_histogram( + "rush_self_metrics_write_duration_ms", + &[("stage", stage), ("table", table)], + started.elapsed().as_secs_f64() * 1000.0, + ); + if let Err(error) = result { + tracing::warn!(%error, stage, table, "stats engine: metrics persistence failed"); + false + } else { + metrics.set_gauge( + "rush_self_metrics_last_success_timestamp_seconds", + &[("stage", stage), ("table", table)], + chrono::Utc::now().timestamp() as f64, + ); + true + } +} + +async fn query_count( + ch: &Client, + sql: &str, + metrics: &SelfMetrics, + probe: &'static str, +) -> Option { + let started = Instant::now(); + let value = ch + .query(sql) .fetch_one::() .await - .map(|r| r.count) - .unwrap_or(0) + .ok() + .map(|r| r.count); + record_ch_probe(metrics, probe, started, value.is_some()); + value } -async fn query_bytes(ch: &Client, sql: &str) -> u64 { - ch.query(sql) +async fn query_bytes( + ch: &Client, + sql: &str, + metrics: &SelfMetrics, + probe: &'static str, +) -> Option { + let started = Instant::now(); + let value = ch + .query(sql) .fetch_one::() .await - .map(|r| r.total) - .unwrap_or(0) + .ok() + .map(|r| r.total); + record_ch_probe(metrics, probe, started, value.is_some()); + value } #[cfg(test)] mod tests { use super::*; + #[tokio::test] + async fn failed_probe_is_invalid_not_zero_and_preserves_last_valid_health_value() { + use clickhouse::test::{Mock, handlers}; + let mock = Mock::new(); + let ch = Client::default() + .with_url(mock.url()) + .with_validation(false); + let metrics = SelfMetrics::new(); + mock.add(handlers::failure( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + )); + assert_eq!( + query_count(&ch, "SELECT count() AS count", &metrics, "logs").await, + None + ); + assert!( + metrics + .render_prometheus() + .contains("rush_ch_health_probe_valid{probe=\"logs\"} 0") + ); + mock.add(handlers::provide([F64Row { v: 42.0 }])); + run_ch_probe(&ch, &metrics, "rush_ch_active_queries", "SELECT 42.0 AS v").await; + mock.add(handlers::failure( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + )); + run_ch_probe(&ch, &metrics, "rush_ch_active_queries", "SELECT 42.0 AS v").await; + let text = metrics.render_prometheus(); + assert!(text.contains("rush_ch_active_queries 42")); + assert!(text.contains("rush_ch_health_probe_valid{probe=\"rush_ch_active_queries\"} 0")); + assert!(text.contains( + "rush_ch_health_probe_last_success_timestamp_seconds{probe=\"rush_ch_active_queries\"}" + )); + } + + #[tokio::test] + async fn snapshot_reports_partial_persistence_failure_and_still_attempts_other_table() { + use clickhouse::test::{Mock, handlers}; + let mock = Mock::new(); + let ch = Client::default() + .with_url(mock.url()) + .with_validation(false); + let metrics = SelfMetrics::new(); + metrics.set_gauge("test_gauge", &[], 1.0); + metrics.inc_counter("test_counter", &[], 1); + mock.add(handlers::failure( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + )); + let sum_write = mock.add(handlers::record_ddl()); + assert!(!self_ingest_snapshot(&ch, &metrics, 1, "test-replica").await); + assert!(sum_write.query().await.contains("INSERT INTO metrics_sum")); + let text = metrics.render_prometheus(); + assert!(text.contains("rush_self_metrics_writes_total{outcome=\"error\",stage=\"snapshot\",table=\"metrics_gauge\"} 1")); + assert!(text.contains("rush_self_metrics_writes_total{outcome=\"ok\",stage=\"snapshot\",table=\"metrics_sum\"} 1")); + assert!(!text.contains( + "last_success_timestamp_seconds{stage=\"snapshot\",table=\"metrics_gauge\"}" + )); + } + #[test] fn ch_map_literal_render() { assert_eq!(labels_to_ch_map(&[]), "{}"); From 457c88755100b3b44f32ee2cf3071147969ededf Mon Sep 17 00:00:00 2001 From: Mike Zupan Date: Sun, 15 Nov 2026 13:54:16 -0800 Subject: [PATCH 2/5] Measure rule evaluation outcomes and export worker health --- src/anomaly_engine.rs | 42 ++++++++++-- src/bin/anomaly_engine.rs | 8 ++- src/clickhouse_config.rs | 12 ++++ src/engine_metrics.rs | 130 ++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 + src/main.rs | 1 + src/monitor_engine.rs | 28 ++++++-- src/self_metrics.rs | 7 ++ src/shutdown.rs | 5 ++ src/siem_engine.rs | 26 ++++++-- src/slo_engine.rs | 11 +++- src/worker_metrics.rs | 103 ++++++++++++++++++++++++++++++ 12 files changed, 356 insertions(+), 19 deletions(-) create mode 100644 src/engine_metrics.rs create mode 100644 src/worker_metrics.rs diff --git a/src/anomaly_engine.rs b/src/anomaly_engine.rs index 8f4b113..c5c4466 100644 --- a/src/anomaly_engine.rs +++ b/src/anomaly_engine.rs @@ -1,5 +1,6 @@ use crate::alert_engine::SmtpConfig; use crate::clickhouse_config::ConfigDb; +use crate::engine_metrics::{EvaluationObservation, EvaluationOutcome}; use crate::models::anomaly::AnomalyRule; use crate::query_builder::ERROR_SPAN; use chrono::DurationRound; @@ -192,6 +193,7 @@ async fn eval_anomaly_rules( let now = chrono::Utc::now(); let now_str = now.format("%Y-%m-%dT%H:%M:%SZ").to_string(); let due_rules = config_db.get_due_anomaly_rules(&now_str).await?; + crate::engine_metrics::due(config_db.self_metrics(), "anomaly_engine", due_rules.len()); if due_rules.is_empty() { tracing::debug!(engine = "anomaly", "tick -- no rules due"); @@ -206,6 +208,8 @@ async fn eval_anomaly_rules( // Prefetch every due rule's data concurrently — the fetches are independent // reads, so the tick pays for the slowest fetch instead of the sum of all. let fetched = futures_util::future::join_all(due_rules.iter().map(|rule| async move { + let mut observation = + EvaluationObservation::new(config_db.self_metrics(), "anomaly_engine"); let fetched = crate::query_governor::run_background(&rule.tenant_id, async { match rule.source.as_str() { "apm" => Some(fetch_apm_data(read_ch, rule, &now).await), @@ -214,12 +218,17 @@ async fn eval_anomaly_rules( } }) .await; - match fetched { + let rejected = fetched.is_err(); + let result = match fetched { Ok(result) => result, - Err(error) => Some(Err(anyhow::anyhow!( - "background admission rejected: {error:?}" - ))), - } + Err(error) => { + observation.finish(EvaluationOutcome::AdmissionRejected); + Some(Err(anyhow::anyhow!( + "background admission rejected: {error:?}" + ))) + } + }; + (result, observation, rejected) })) .await; @@ -227,7 +236,11 @@ async fn eval_anomaly_rules( // at the end of the tick (previously one INSERT per series per rule). let mut eval_log_values: Vec = Vec::new(); - for (rule, prefetched) in due_rules.into_iter().zip(fetched) { + let mut failures = 0; + for (rule, (prefetched, mut observation, rejected)) in due_rules.into_iter().zip(fetched) { + if !rejected { + observation.finish(EvaluationOutcome::Error); + } tracing::debug!( engine = "anomaly", rule_name = %rule.name, @@ -239,6 +252,8 @@ async fn eval_anomaly_rules( let all_series = match prefetched { Some(r) => r, None => { + observation.finish(EvaluationOutcome::InvalidConfig); + failures += 1; tracing::warn!(engine = "anomaly", rule_id = %rule.id, source = %rule.source, "unknown data source"); continue; } @@ -251,9 +266,11 @@ async fn eval_anomaly_rules( config_db .update_anomaly_state(&rule.id, &rule.tenant_id, "no_data", &now_str, None) .await?; + observation.finish(EvaluationOutcome::NoData); continue; } Err(e) => { + failures += 1; tracing::warn!(error = %e, engine = "anomaly", rule_id = %rule.id, "data fetch failed"); config_db .update_anomaly_state(&rule.id, &rule.tenant_id, "no_data", &now_str, None) @@ -391,6 +408,7 @@ async fn eval_anomaly_rules( .update_anomaly_state(&rule.id, &rule.tenant_id, new_state, &now_str, None) .await?; } + observation.finish(EvaluationOutcome::Completed); } // Flush all per-series eval logs in one INSERT (previously one per series). @@ -400,10 +418,22 @@ async fn eval_anomaly_rules( eval_log_values.join(", ") ); if let Err(e) = write_ch.query(&sql).execute().await { + failures += 1; + if let Some(metrics) = config_db.self_metrics() { + metrics.inc_counter( + "rush_engine_persistence_failures_total", + &[("engine", "anomaly_engine"), ("operation", "eval_log")], + 1, + ); + } tracing::warn!(error = %e, engine = "anomaly", rows = eval_log_values.len(), "failed to write eval log batch"); } } + anyhow::ensure!( + failures == 0, + "{failures} anomaly evaluation operation(s) failed" + ); Ok(()) } diff --git a/src/bin/anomaly_engine.rs b/src/bin/anomaly_engine.rs index 35adce4..7e589c5 100644 --- a/src/bin/anomaly_engine.rs +++ b/src/bin/anomaly_engine.rs @@ -75,9 +75,10 @@ async fn main() -> anyhow::Result<()> { std::env::var("RUSH_PROM_BASE_URL").unwrap_or_else(|_| "http://localhost:8080".to_string()); tracing::info!("wide-anomaly-engine starting"); - // The standalone engine has no /metrics endpoint; give it a private registry so the - // engine-loop instrumentation still works (record_engine just updates in-memory atomics). + // The standalone worker exports the same bounded metrics as the API process. let self_metrics = Arc::new(rush_api::self_metrics::SelfMetrics::new()); + config_db.attach_metrics(self_metrics.clone()); + rush_api::process_metrics::spawn(self_metrics.clone()); let query_limits = rush_api::query_governor::QueryGovernorConfig::at_startup( config_db .get_setting(rush_api::query_governor::QUERY_LIMITS_SETTING_KEY) @@ -94,6 +95,8 @@ async fn main() -> anyhow::Result<()> { // Takes the same lease as the in-process engine, so this deployment and any // API replica can never both evaluate rules. let shutdown = rush_api::shutdown::ShutdownController::new(); + let _metrics_server = + rush_api::worker_metrics::spawn(self_metrics.clone(), shutdown.clone()).await?; let election = rush_api::leader::LeaderElection::from_env( config_db.client.clone(), &rush_api::stats_engine::configured_instance_id(), @@ -120,6 +123,7 @@ async fn main() -> anyhow::Result<()> { } } + shutdown.finish(); Ok(()) } diff --git a/src/clickhouse_config.rs b/src/clickhouse_config.rs index 725dca5..4f8f815 100644 --- a/src/clickhouse_config.rs +++ b/src/clickhouse_config.rs @@ -87,6 +87,15 @@ struct SavedQueryRow { } impl ConfigDb { + /// Attach the shared process registry before starting background work. + pub fn attach_metrics(&self, metrics: std::sync::Arc) { + let _ = self.metrics.set(metrics); + } + + pub fn self_metrics(&self) -> Option<&crate::self_metrics::SelfMetrics> { + self.metrics.get().map(std::sync::Arc::as_ref) + } + pub async fn create_investigation_session( &self, id: &str, @@ -2443,6 +2452,7 @@ mod sql_shape_tests { } pub struct ConfigDb { + metrics: std::sync::OnceLock>, pub client: Client, /// Tenant name-or-id → flags. Negative results are cached too so unknown /// tenants on the ingest path don't hammer ClickHouse. @@ -2635,6 +2645,7 @@ impl ConfigDb { .with_user(user) .with_password(password); let db = Self { + metrics: std::sync::OnceLock::new(), client, tenant_cache: DashMap::new(), ingest_auth_cache: DashMap::new(), @@ -12407,6 +12418,7 @@ mod session_tenant_regressions { fn config(mock: &Mock) -> ConfigDb { ConfigDb { + metrics: std::sync::OnceLock::new(), client: Client::default() .with_url(mock.url()) .with_compression(clickhouse::Compression::None), diff --git a/src/engine_metrics.rs b/src/engine_metrics.rs new file mode 100644 index 0000000..0d560fe --- /dev/null +++ b/src/engine_metrics.rs @@ -0,0 +1,130 @@ +//! Bounded evaluation outcomes, including cancelled work and successful no-data reads. +use crate::self_metrics::SelfMetrics; +use std::time::Instant; + +#[derive(Clone, Copy)] +pub enum EvaluationOutcome { + Completed, + NoData, + Error, + AdmissionRejected, + InvalidConfig, +} + +pub struct EvaluationObservation<'a> { + metrics: Option<&'a SelfMetrics>, + engine: &'static str, + started: Instant, + outcome: &'static str, +} + +impl<'a> EvaluationObservation<'a> { + pub fn new(metrics: Option<&'a SelfMetrics>, engine: &'static str) -> Self { + let engine = match engine { + "monitor_engine" | "siem_engine" | "slo_engine" | "anomaly_engine" => engine, + _ => "other", + }; + if let Some(metrics) = metrics { + metrics.add_gauge( + "rush_engine_evaluations_inflight", + &[("engine", engine)], + 1.0, + ); + } + Self { + metrics, + engine, + started: Instant::now(), + outcome: "cancelled", + } + } + pub fn finish(&mut self, outcome: EvaluationOutcome) { + self.outcome = match outcome { + EvaluationOutcome::Completed => "ok", + EvaluationOutcome::NoData => "no_data", + EvaluationOutcome::Error => "error", + EvaluationOutcome::AdmissionRejected => "admission_rejected", + EvaluationOutcome::InvalidConfig => "invalid_config", + }; + } +} + +impl Drop for EvaluationObservation<'_> { + fn drop(&mut self) { + if let Some(metrics) = self.metrics { + let labels = [("engine", self.engine), ("outcome", self.outcome)]; + metrics.add_gauge( + "rush_engine_evaluations_inflight", + &[("engine", self.engine)], + -1.0, + ); + metrics.inc_counter("rush_engine_evaluations_total", &labels, 1); + metrics.observe_histogram( + "rush_engine_evaluation_duration_ms", + &labels, + self.started.elapsed().as_secs_f64() * 1000.0, + ); + if matches!(self.outcome, "ok" | "no_data") { + metrics.set_gauge( + "rush_engine_evaluation_last_success_timestamp_seconds", + &[("engine", self.engine)], + chrono::Utc::now().timestamp() as f64, + ); + } + } + } +} + +pub fn due(metrics: Option<&SelfMetrics>, engine: &'static str, count: usize) { + if let Some(metrics) = metrics { + metrics.set_gauge( + "rush_engine_evaluations_due", + &[("engine", engine)], + count as f64, + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + #[tokio::test] + async fn cancellation_releases_inflight_and_is_not_a_success() { + let metrics = Arc::new(SelfMetrics::new()); + let registry = metrics.clone(); + let (tx, rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let _observation = EvaluationObservation::new(Some(®istry), "monitor_engine"); + tx.send(()).unwrap(); + std::future::pending::<()>().await; + }); + rx.await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + let text = metrics.render_prometheus(); + assert!(text.contains("rush_engine_evaluations_inflight{engine=\"monitor_engine\"} 0")); + assert!(text.contains( + "rush_engine_evaluations_total{engine=\"monitor_engine\",outcome=\"cancelled\"} 1" + )); + assert!(!text.contains("last_success")); + } + #[test] + fn failed_work_does_not_advance_last_success_and_labels_are_bounded() { + let metrics = SelfMetrics::new(); + { + let mut work = EvaluationObservation::new(Some(&metrics), "slo_engine"); + work.finish(EvaluationOutcome::NoData); + } + { + let mut work = EvaluationObservation::new(Some(&metrics), "raw-rule-name"); + work.finish(EvaluationOutcome::Error); + } + let text = metrics.render_prometheus(); + assert!(text.contains( + "rush_engine_evaluation_last_success_timestamp_seconds{engine=\"slo_engine\"}" + )); + assert!(!text.contains("last_success_timestamp_seconds{engine=\"other\"}")); + assert!(!text.contains("raw-rule-name")); + } +} diff --git a/src/lib.rs b/src/lib.rs index 0e3c529..9d195b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ pub mod config; pub mod cors; pub mod detection_query; pub mod edition; +pub mod engine_metrics; pub mod eval_state; pub mod github_repository_policy; pub mod handlers; @@ -45,6 +46,7 @@ pub mod spool; pub mod stats_engine; pub mod usage_accumulator; pub mod usage_tracker; +pub mod worker_metrics; use clickhouse::Client; use clickhouse::query::Query; diff --git a/src/main.rs b/src/main.rs index b1e60cd..0140418 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2584,6 +2584,7 @@ async fn main() -> anyhow::Result<()> { // outbox degradation is visible even if no one has scraped `/metrics` yet. let self_metrics: std::sync::Arc = std::sync::Arc::new(rush_api::self_metrics::SelfMetrics::new()); + config_db.attach_metrics(self_metrics.clone()); let query_governor_config = QueryGovernorConfig::at_startup( config_db .get_setting(QUERY_LIMITS_SETTING_KEY) diff --git a/src/monitor_engine.rs b/src/monitor_engine.rs index 9d70e56..ae7acc7 100644 --- a/src/monitor_engine.rs +++ b/src/monitor_engine.rs @@ -1,3 +1,4 @@ +use crate::engine_metrics::{EvaluationObservation, EvaluationOutcome}; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; use std::time::Instant; @@ -342,8 +343,11 @@ async fn run_evaluation_cycle( let now_str_ref = now_str.as_str(); let monitor_states_ref = &monitor_states; - let outcomes: Vec<(String, u64, bool)> = + crate::engine_metrics::due(config_db.self_metrics(), "monitor_engine", jobs.len()); + let outcomes: Vec<(String, u64, bool, bool)> = futures_util::stream::iter(jobs.into_iter().map(|(monitor, should_flush)| async move { + let mut observation = + EvaluationObservation::new(config_db.self_metrics(), "monitor_engine"); let result = match crate::query_governor::run_background( &monitor.tenant_id, evaluate_monitor( @@ -361,9 +365,20 @@ async fn run_evaluation_cycle( ) .await { - Ok(result) => result, - Err(error) => Err(anyhow::anyhow!("background admission rejected: {error:?}")), + Ok(result) => { + observation.finish(if result.is_ok() { + EvaluationOutcome::Completed + } else { + EvaluationOutcome::Error + }); + result + } + Err(error) => { + observation.finish(EvaluationOutcome::AdmissionRejected); + Err(anyhow::anyhow!("background admission rejected: {error:?}")) + } }; + let failed = result.is_err(); let (changes, persisted) = match result { Ok(cp) => cp, Err(e) => { @@ -388,18 +403,21 @@ async fn run_evaluation_cycle( .await } }; - (monitor.id, changes, persisted) + (monitor.id, changes, persisted, failed) })) .buffer_unordered(ENGINE_CONCURRENCY) .collect() .await; let mut state_changes: u64 = 0; - for (id, changes, persisted) in outcomes { + let mut failures = 0; + for (id, changes, persisted, failed) in outcomes { + failures += usize::from(failed); state_changes += changes; eval_state.record(id, now, persisted); } + anyhow::ensure!(failures == 0, "{failures} monitor evaluation job(s) failed"); Ok((evaluated, state_changes)) } diff --git a/src/self_metrics.rs b/src/self_metrics.rs index 16e6718..b6a0e23 100644 --- a/src/self_metrics.rs +++ b/src/self_metrics.rs @@ -409,6 +409,13 @@ impl SelfMetrics { .map(|d| d.as_secs()) .unwrap_or(0); self.set_gauge("rush_engine_last_run_timestamp", &labels, now as f64); + if ok { + self.set_gauge( + "rush_engine_last_success_timestamp_seconds", + &labels, + now as f64, + ); + } } /// Convenience: record one log/span SEARCH and update all search self-metrics. diff --git a/src/shutdown.rs b/src/shutdown.rs index 3eac78a..a4c8d0e 100644 --- a/src/shutdown.rs +++ b/src/shutdown.rs @@ -55,6 +55,11 @@ impl ShutdownController { .await; } + pub async fn wait_for_finish(&self) { + self.wait_until(|phase| phase == ShutdownPhase::Finished) + .await; + } + pub async fn wait_for_drain(&self) { self.wait_until(|phase| phase >= ShutdownPhase::Draining) .await; diff --git a/src/siem_engine.rs b/src/siem_engine.rs index e4f27d1..613afb6 100644 --- a/src/siem_engine.rs +++ b/src/siem_engine.rs @@ -1,4 +1,5 @@ use crate::clickhouse_config::ConfigDb; +use crate::engine_metrics::{EvaluationObservation, EvaluationOutcome}; use crate::models::detection::DetectionRule; use clickhouse::Client; use std::sync::Arc; @@ -59,6 +60,7 @@ async fn run_detection_cycle( let rules = config_db.list_enabled_detection_rules().await?; if rules.is_empty() { + crate::engine_metrics::due(config_db.self_metrics(), "siem_engine", 0); tracing::debug!(engine = "siem", "tick -- no enabled detection rules"); return Ok(()); } @@ -76,11 +78,14 @@ async fn run_detection_cycle( }) .collect(); + crate::engine_metrics::due(config_db.self_metrics(), "siem_engine", jobs.len()); let evaluated = jobs.len() as u32; let now_str_ref = now_str.as_str(); - let outcomes: Vec<(String, bool, bool)> = + let outcomes: Vec<(String, bool, bool, bool)> = futures_util::stream::iter(jobs.into_iter().map(|(rule, should_flush)| async move { + let mut observation = + EvaluationObservation::new(config_db.self_metrics(), "siem_engine"); let evaluated = crate::query_governor::run_background( &rule.tenant_id, evaluate_rule( @@ -95,8 +100,12 @@ async fn run_detection_cycle( ) .await; match evaluated { - Ok(Ok((did_fire, persisted))) => (rule.id, did_fire, persisted), + Ok(Ok((did_fire, persisted))) => { + observation.finish(EvaluationOutcome::Completed); + (rule.id, did_fire, persisted, false) + } Ok(Err(e)) => { + observation.finish(EvaluationOutcome::Error); tracing::warn!( error = %e, engine = "siem", @@ -104,9 +113,10 @@ async fn run_detection_cycle( rule_id = %rule.id, "rule evaluation failed" ); - (rule.id, false, false) + (rule.id, false, false, true) } Err(e) => { + observation.finish(EvaluationOutcome::AdmissionRejected); tracing::warn!( error = ?e, engine = "siem", @@ -114,7 +124,7 @@ async fn run_detection_cycle( rule_id = %rule.id, "rule evaluation admission rejected" ); - (rule.id, false, false) + (rule.id, false, false, true) } } })) @@ -123,7 +133,9 @@ async fn run_detection_cycle( .await; let mut fired = 0u32; - for (id, did_fire, persisted) in outcomes { + let mut failures = 0; + for (id, did_fire, persisted, failed) in outcomes { + failures += usize::from(failed); if did_fire { fired += 1; } @@ -141,6 +153,10 @@ async fn run_detection_cycle( tracing::debug!(engine = "siem", "tick -- no rules due"); } + anyhow::ensure!( + failures == 0, + "{failures} detection evaluation job(s) failed" + ); Ok(()) } diff --git a/src/slo_engine.rs b/src/slo_engine.rs index 1c1de9b..404a04c 100644 --- a/src/slo_engine.rs +++ b/src/slo_engine.rs @@ -1,4 +1,5 @@ use crate::clickhouse_config::ConfigDb; +use crate::engine_metrics::{EvaluationObservation, EvaluationOutcome}; use crate::models::query::{Filter, FilterOp}; use crate::query_builder::{QueryClauses, build_metrics_where_clause, build_where_clause}; use clickhouse::Client; @@ -405,9 +406,12 @@ async fn eval_slos( }) .collect(); + crate::engine_metrics::due(config_db.self_metrics(), "slo_engine", jobs.len()); let now_str_ref = now_str.as_str(); let outcomes: Vec<(String, bool, bool)> = futures_util::stream::iter(jobs.into_iter().map(|(slo, should_flush)| async move { + let mut observation = + EvaluationObservation::new(config_db.self_metrics(), "slo_engine"); let evaluated = crate::query_governor::run_background( &slo.tenant_id, eval_one_slo( @@ -423,12 +427,17 @@ async fn eval_slos( ) .await; let (persisted, succeeded) = match evaluated { - Ok(Ok(p)) => (p, true), + Ok(Ok(p)) => { + observation.finish(EvaluationOutcome::Completed); + (p, true) + } Ok(Err(e)) => { + observation.finish(EvaluationOutcome::Error); tracing::warn!("slo {}: evaluation error: {e}", slo.id); (false, false) } Err(e) => { + observation.finish(EvaluationOutcome::AdmissionRejected); tracing::warn!("slo {}: background admission rejected: {e:?}", slo.id); (false, false) } diff --git a/src/worker_metrics.rs b/src/worker_metrics.rs new file mode 100644 index 0000000..7f0d6e9 --- /dev/null +++ b/src/worker_metrics.rs @@ -0,0 +1,103 @@ +//! A small scrape listener for worker processes that do not serve the API. +use crate::{self_metrics::SelfMetrics, shutdown::ShutdownController}; +use axum::{Router, extract::State, routing::get}; +use std::sync::Arc; + +pub const DEFAULT_WORKER_METRICS_ADDR: &str = "0.0.0.0:9090"; + +/// An explicit empty address disables the listener. Bind errors are startup +/// errors, rather than silently running a worker without its configured metrics. +pub async fn spawn( + metrics: Arc, + shutdown: ShutdownController, +) -> anyhow::Result>> { + let address = std::env::var("RUSH_WORKER_METRICS_ADDR") + .unwrap_or_else(|_| DEFAULT_WORKER_METRICS_ADDR.to_string()); + if address.trim().is_empty() { + return Ok(None); + } + let listener = tokio::net::TcpListener::bind(address.trim()).await?; + Ok(Some(spawn_listener(listener, metrics, shutdown))) +} + +fn spawn_listener( + listener: tokio::net::TcpListener, + metrics: Arc, + shutdown: ShutdownController, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + if let Err(error) = axum::serve(listener, router(metrics)) + .with_graceful_shutdown(async move { shutdown.wait_for_finish().await }) + .await + { + tracing::error!(%error, "worker metrics listener failed"); + } + }) +} + +fn router(metrics: Arc) -> Router { + Router::new() + .route( + "/metrics", + get(|State(metrics): State>| async move { + ( + [("content-type", "text/plain; version=0.0.4; charset=utf-8")], + metrics.render_prometheus(), + ) + }), + ) + .with_state(metrics) +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn worker_exposes_metrics_without_serving_api_routes() { + let metrics = Arc::new(SelfMetrics::new()); + metrics.inc_counter( + "rush_ingest_replay_batches_total", + &[("outcome", "committed")], + 1, + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let shutdown = ShutdownController::new(); + let server = spawn_listener(listener, metrics, shutdown.clone()); + shutdown.request(); // metrics remain available while the worker drains + + let client = reqwest::Client::new(); + let response = client + .get(format!("http://{address}/metrics")) + .send() + .await + .unwrap(); + assert!( + response.headers()["content-type"] + .to_str() + .unwrap() + .starts_with("text/plain") + ); + assert!( + response + .text() + .await + .unwrap() + .contains("rush_ingest_replay_batches_total") + ); + assert_eq!( + client + .get(format!("http://{address}/api/v1/tenants")) + .send() + .await + .unwrap() + .status(), + 404 + ); + shutdown.finish(); + tokio::time::timeout(std::time::Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); + } +} From 54b3b86d95d3e7ee912d513801580299fa2eab61 Mon Sep 17 00:00:00 2001 From: Mike Zupan Date: Sun, 15 Nov 2026 13:57:16 -0800 Subject: [PATCH 3/5] Measure notification delivery and log persistence outcomes --- src/alert_engine.rs | 136 +++++++++++++++++++++--------- src/bin/anomaly_engine.rs | 8 +- src/clickhouse_config.rs | 12 +++ src/handlers/alerts.rs | 42 ++++++---- src/lib.rs | 2 + src/main.rs | 1 + src/notification_metrics.rs | 161 ++++++++++++++++++++++++++++++++++++ src/shutdown.rs | 5 ++ src/worker_metrics.rs | 103 +++++++++++++++++++++++ 9 files changed, 415 insertions(+), 55 deletions(-) create mode 100644 src/notification_metrics.rs create mode 100644 src/worker_metrics.rs diff --git a/src/alert_engine.rs b/src/alert_engine.rs index 32fe59a..77e0114 100644 --- a/src/alert_engine.rs +++ b/src/alert_engine.rs @@ -348,9 +348,28 @@ pub async fn notify_channels( ) { for channel_id in channel_ids { let channel = match config_db.get_channel(channel_id, tenant_id).await { - Ok(Some(channel)) if channel.enabled => channel, - Ok(_) => continue, + Ok(Some(channel)) if channel.enabled => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + notification.source, + "ready", + ); + channel + } + Ok(_) => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + notification.source, + "skipped", + ); + continue; + } Err(error) => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + notification.source, + "error", + ); tracing::warn!(source = notification.source, %channel_id, %error, "failed to load notification channel"); continue; } @@ -360,22 +379,27 @@ pub async fn notify_channels( } else { notification.alert_id }; - let result = send_channel_notification_with_fields( - &channel, - notification.message, - notification.alert_name, - notification.alert_state, - notification.value, - notification.threshold, - notification.signal_type, - notification.condition_op, - notification.description, - notification_id, - notification.runbook_url, - notification.webhook_fields, - http_client, - smtp_config, - smtp_transport, + let result = crate::notification_metrics::observe_delivery( + config_db.self_metrics(), + notification.source, + &channel.channel_type, + send_channel_notification_with_fields( + &channel, + notification.message, + notification.alert_name, + notification.alert_state, + notification.value, + notification.threshold, + notification.signal_type, + notification.condition_op, + notification.description, + notification_id, + notification.runbook_url, + notification.webhook_fields, + http_client, + smtp_config, + smtp_transport, + ), ) .await; let (status, error_msg) = match &result { @@ -385,7 +409,7 @@ pub async fn notify_channels( ("failed", error.clone()) } }; - let _ = config_db + let logged = config_db .create_notification_log( channel_id, tenant_id, @@ -396,6 +420,11 @@ pub async fn notify_channels( &error_msg, ) .await; + crate::notification_metrics::record_log_write( + config_db.self_metrics(), + notification.source, + logged.is_ok(), + ); } } @@ -899,25 +928,53 @@ async fn eval_alert_rule( serde_json::from_str(&rule.notification_channel_ids).unwrap_or_default(); let alert_state_str = if triggered { "FIRING" } else { "RESOLVED" }; for channel_id in &channel_ids { - if let Ok(Some(channel)) = config_db.get_channel_by_id(channel_id).await { - if !channel.enabled { + let channel = match config_db.get_channel_by_id(channel_id).await { + Ok(Some(channel)) if channel.enabled => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + "alert_rule", + "ready", + ); + channel + } + Ok(_) => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + "alert_rule", + "skipped", + ); continue; } - let result = send_channel_notification( - &channel, - &message, - &rule.name, - alert_state_str, - value, - threshold, - &rule.signal_type, - &rule.condition_op, - &rule.description, - &rule.id, - &rule.runbook_url, - http_client, - smtp_config, - smtp_transport, + Err(_) => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + "alert_rule", + "error", + ); + continue; + } + }; + { + let result = crate::notification_metrics::observe_delivery( + config_db.self_metrics(), + "alert_rule", + &channel.channel_type, + send_channel_notification( + &channel, + &message, + &rule.name, + alert_state_str, + value, + threshold, + &rule.signal_type, + &rule.condition_op, + &rule.description, + &rule.id, + &rule.runbook_url, + http_client, + smtp_config, + smtp_transport, + ), ) .await; @@ -933,7 +990,7 @@ async fn eval_alert_rule( } }; - let _ = config_db + let logged = config_db .create_notification_log( channel_id, &channel.tenant_id, @@ -944,6 +1001,11 @@ async fn eval_alert_rule( &error_msg, ) .await; + crate::notification_metrics::record_log_write( + config_db.self_metrics(), + "alert_rule", + logged.is_ok(), + ); } } diff --git a/src/bin/anomaly_engine.rs b/src/bin/anomaly_engine.rs index 35adce4..7e589c5 100644 --- a/src/bin/anomaly_engine.rs +++ b/src/bin/anomaly_engine.rs @@ -75,9 +75,10 @@ async fn main() -> anyhow::Result<()> { std::env::var("RUSH_PROM_BASE_URL").unwrap_or_else(|_| "http://localhost:8080".to_string()); tracing::info!("wide-anomaly-engine starting"); - // The standalone engine has no /metrics endpoint; give it a private registry so the - // engine-loop instrumentation still works (record_engine just updates in-memory atomics). + // The standalone worker exports the same bounded metrics as the API process. let self_metrics = Arc::new(rush_api::self_metrics::SelfMetrics::new()); + config_db.attach_metrics(self_metrics.clone()); + rush_api::process_metrics::spawn(self_metrics.clone()); let query_limits = rush_api::query_governor::QueryGovernorConfig::at_startup( config_db .get_setting(rush_api::query_governor::QUERY_LIMITS_SETTING_KEY) @@ -94,6 +95,8 @@ async fn main() -> anyhow::Result<()> { // Takes the same lease as the in-process engine, so this deployment and any // API replica can never both evaluate rules. let shutdown = rush_api::shutdown::ShutdownController::new(); + let _metrics_server = + rush_api::worker_metrics::spawn(self_metrics.clone(), shutdown.clone()).await?; let election = rush_api::leader::LeaderElection::from_env( config_db.client.clone(), &rush_api::stats_engine::configured_instance_id(), @@ -120,6 +123,7 @@ async fn main() -> anyhow::Result<()> { } } + shutdown.finish(); Ok(()) } diff --git a/src/clickhouse_config.rs b/src/clickhouse_config.rs index 725dca5..4f8f815 100644 --- a/src/clickhouse_config.rs +++ b/src/clickhouse_config.rs @@ -87,6 +87,15 @@ struct SavedQueryRow { } impl ConfigDb { + /// Attach the shared process registry before starting background work. + pub fn attach_metrics(&self, metrics: std::sync::Arc) { + let _ = self.metrics.set(metrics); + } + + pub fn self_metrics(&self) -> Option<&crate::self_metrics::SelfMetrics> { + self.metrics.get().map(std::sync::Arc::as_ref) + } + pub async fn create_investigation_session( &self, id: &str, @@ -2443,6 +2452,7 @@ mod sql_shape_tests { } pub struct ConfigDb { + metrics: std::sync::OnceLock>, pub client: Client, /// Tenant name-or-id → flags. Negative results are cached too so unknown /// tenants on the ingest path don't hammer ClickHouse. @@ -2635,6 +2645,7 @@ impl ConfigDb { .with_user(user) .with_password(password); let db = Self { + metrics: std::sync::OnceLock::new(), client, tenant_cache: DashMap::new(), ingest_auth_cache: DashMap::new(), @@ -12407,6 +12418,7 @@ mod session_tenant_regressions { fn config(mock: &Mock) -> ConfigDb { ConfigDb { + metrics: std::sync::OnceLock::new(), client: Client::default() .with_url(mock.url()) .with_compression(clickhouse::Compression::None), diff --git a/src/handlers/alerts.rs b/src/handlers/alerts.rs index 25d909a..09cc4df 100644 --- a/src/handlers/alerts.rs +++ b/src/handlers/alerts.rs @@ -465,21 +465,26 @@ pub async fn test_channel( }; let http_client = reqwest::Client::new(); - let result = crate::alert_engine::send_channel_notification( - &channel, - &test_message, - "Test Alert", - "TEST", - 0.0, - 0.0, - "", - "", - "This is a test notification from Rush Observability.", - "", - "", - &http_client, - &smtp_config, - &smtp_transport, + let result = crate::notification_metrics::observe_delivery( + Some(&state.self_metrics), + "test", + &channel.channel_type, + crate::alert_engine::send_channel_notification( + &channel, + &test_message, + "Test Alert", + "TEST", + 0.0, + 0.0, + "", + "", + "This is a test notification from Rush Observability.", + "", + "", + &http_client, + &smtp_config, + &smtp_transport, + ), ) .await; @@ -501,7 +506,7 @@ pub async fn test_channel( ) .await; - let _ = state + let logged = state .config_db .create_notification_log( &id, @@ -513,6 +518,11 @@ pub async fn test_channel( &error_msg, ) .await; + crate::notification_metrics::record_log_write( + Some(&state.self_metrics), + "test", + logged.is_ok(), + ); match result { Ok(()) => Ok(Json( diff --git a/src/lib.rs b/src/lib.rs index 0e3c529..252b4c2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ pub mod metric_firewall; pub mod migrations; pub mod models; pub mod monitor_engine; +pub mod notification_metrics; pub mod object_store_spool; pub mod outbound; pub mod pagination; @@ -45,6 +46,7 @@ pub mod spool; pub mod stats_engine; pub mod usage_accumulator; pub mod usage_tracker; +pub mod worker_metrics; use clickhouse::Client; use clickhouse::query::Query; diff --git a/src/main.rs b/src/main.rs index b1e60cd..0140418 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2584,6 +2584,7 @@ async fn main() -> anyhow::Result<()> { // outbox degradation is visible even if no one has scraped `/metrics` yet. let self_metrics: std::sync::Arc = std::sync::Arc::new(rush_api::self_metrics::SelfMetrics::new()); + config_db.attach_metrics(self_metrics.clone()); let query_governor_config = QueryGovernorConfig::at_startup( config_db .get_setting(QUERY_LIMITS_SETTING_KEY) diff --git a/src/notification_metrics.rs b/src/notification_metrics.rs new file mode 100644 index 0000000..7d97a43 --- /dev/null +++ b/src/notification_metrics.rs @@ -0,0 +1,161 @@ +//! Notification health without destination, tenant, alert or credential labels. +use crate::self_metrics::SelfMetrics; +use std::{future::Future, time::Instant}; + +fn source(value: &str) -> &'static str { + match value { + "monitor" => "monitor", + "slo" => "slo", + "anomaly" => "anomaly", + "detection" => "detection", + "alert_rule" => "alert_rule", + "test" => "test", + _ => "other", + } +} +fn channel_type(value: &str) -> &'static str { + match value { + "email" => "email", + "slack" => "slack", + "slack_app" => "slack_app", + "webhook" => "webhook", + "pagerduty" => "pagerduty", + "rootly" => "rootly", + "discord" => "discord", + "alertmanager" => "alertmanager", + _ => "other", + } +} +struct DeliveryObservation<'a> { + metrics: Option<&'a SelfMetrics>, + source: &'static str, + channel: &'static str, + started: Instant, + outcome: &'static str, +} +impl Drop for DeliveryObservation<'_> { + fn drop(&mut self) { + if let Some(metrics) = self.metrics { + let labels = [("source", self.source), ("channel_type", self.channel)]; + metrics.add_gauge("rush_notification_deliveries_inflight", &labels, -1.0); + let labels = [ + ("source", self.source), + ("channel_type", self.channel), + ("outcome", self.outcome), + ]; + metrics.inc_counter("rush_notification_deliveries_total", &labels, 1); + metrics.observe_histogram( + "rush_notification_delivery_duration_ms", + &labels, + self.started.elapsed().as_secs_f64() * 1000.0, + ); + } + } +} + +pub async fn observe_delivery( + metrics: Option<&SelfMetrics>, + origin: &str, + channel: &str, + work: impl Future>, +) -> Result<(), String> { + let mut observation = DeliveryObservation { + metrics, + source: source(origin), + channel: channel_type(channel), + started: Instant::now(), + outcome: "cancelled", + }; + if let Some(metrics) = metrics { + metrics.add_gauge( + "rush_notification_deliveries_inflight", + &[ + ("source", observation.source), + ("channel_type", observation.channel), + ], + 1.0, + ); + } + let result = work.await; + observation.outcome = if result.is_ok() { "sent" } else { "error" }; + result +} + +pub fn record_lookup(metrics: Option<&SelfMetrics>, origin: &str, outcome: &'static str) { + if let Some(metrics) = metrics { + let outcome = match outcome { + "ready" | "skipped" | "error" => outcome, + _ => "other", + }; + metrics.inc_counter( + "rush_notification_channel_lookups_total", + &[("source", source(origin)), ("outcome", outcome)], + 1, + ); + } +} +pub fn record_log_write(metrics: Option<&SelfMetrics>, origin: &str, ok: bool) { + if let Some(metrics) = metrics { + metrics.inc_counter( + "rush_notification_log_writes_total", + &[ + ("source", source(origin)), + ("outcome", if ok { "ok" } else { "error" }), + ], + 1, + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + #[tokio::test] + async fn delivery_outcomes_are_distinct_and_labels_never_include_error_or_destination() { + let metrics = SelfMetrics::new(); + observe_delivery(Some(&metrics), "monitor", "email", async { Ok(()) }) + .await + .unwrap(); + assert!( + observe_delivery( + Some(&metrics), + "tenant-secret", + "https://secret-destination", + async { Err("credential-secret".into()) } + ) + .await + .is_err() + ); + record_log_write(Some(&metrics), "monitor", false); + let text = metrics.render_prometheus(); + assert!(text.contains("rush_notification_deliveries_total{channel_type=\"email\",outcome=\"sent\",source=\"monitor\"} 1")); + assert!(text.contains("rush_notification_deliveries_total{channel_type=\"other\",outcome=\"error\",source=\"other\"} 1")); + assert!(text.contains( + "rush_notification_log_writes_total{outcome=\"error\",source=\"monitor\"} 1" + )); + assert!(!text.contains("secret")); + } + #[tokio::test] + async fn cancelled_delivery_releases_inflight_without_recording_a_send() { + let metrics = Arc::new(SelfMetrics::new()); + let registry = metrics.clone(); + let (tx, rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + observe_delivery(Some(®istry), "slo", "webhook", async { + tx.send(()).unwrap(); + std::future::pending::>().await + }) + .await + }); + rx.await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + let text = metrics.render_prometheus(); + assert!(text.contains( + "rush_notification_deliveries_inflight{channel_type=\"webhook\",source=\"slo\"} 0" + )); + assert!(text.contains("rush_notification_deliveries_total{channel_type=\"webhook\",outcome=\"cancelled\",source=\"slo\"} 1")); + assert!(!text.contains("outcome=\"sent\"")); + } +} diff --git a/src/shutdown.rs b/src/shutdown.rs index 3eac78a..a4c8d0e 100644 --- a/src/shutdown.rs +++ b/src/shutdown.rs @@ -55,6 +55,11 @@ impl ShutdownController { .await; } + pub async fn wait_for_finish(&self) { + self.wait_until(|phase| phase == ShutdownPhase::Finished) + .await; + } + pub async fn wait_for_drain(&self) { self.wait_until(|phase| phase >= ShutdownPhase::Draining) .await; diff --git a/src/worker_metrics.rs b/src/worker_metrics.rs new file mode 100644 index 0000000..7f0d6e9 --- /dev/null +++ b/src/worker_metrics.rs @@ -0,0 +1,103 @@ +//! A small scrape listener for worker processes that do not serve the API. +use crate::{self_metrics::SelfMetrics, shutdown::ShutdownController}; +use axum::{Router, extract::State, routing::get}; +use std::sync::Arc; + +pub const DEFAULT_WORKER_METRICS_ADDR: &str = "0.0.0.0:9090"; + +/// An explicit empty address disables the listener. Bind errors are startup +/// errors, rather than silently running a worker without its configured metrics. +pub async fn spawn( + metrics: Arc, + shutdown: ShutdownController, +) -> anyhow::Result>> { + let address = std::env::var("RUSH_WORKER_METRICS_ADDR") + .unwrap_or_else(|_| DEFAULT_WORKER_METRICS_ADDR.to_string()); + if address.trim().is_empty() { + return Ok(None); + } + let listener = tokio::net::TcpListener::bind(address.trim()).await?; + Ok(Some(spawn_listener(listener, metrics, shutdown))) +} + +fn spawn_listener( + listener: tokio::net::TcpListener, + metrics: Arc, + shutdown: ShutdownController, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + if let Err(error) = axum::serve(listener, router(metrics)) + .with_graceful_shutdown(async move { shutdown.wait_for_finish().await }) + .await + { + tracing::error!(%error, "worker metrics listener failed"); + } + }) +} + +fn router(metrics: Arc) -> Router { + Router::new() + .route( + "/metrics", + get(|State(metrics): State>| async move { + ( + [("content-type", "text/plain; version=0.0.4; charset=utf-8")], + metrics.render_prometheus(), + ) + }), + ) + .with_state(metrics) +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn worker_exposes_metrics_without_serving_api_routes() { + let metrics = Arc::new(SelfMetrics::new()); + metrics.inc_counter( + "rush_ingest_replay_batches_total", + &[("outcome", "committed")], + 1, + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let shutdown = ShutdownController::new(); + let server = spawn_listener(listener, metrics, shutdown.clone()); + shutdown.request(); // metrics remain available while the worker drains + + let client = reqwest::Client::new(); + let response = client + .get(format!("http://{address}/metrics")) + .send() + .await + .unwrap(); + assert!( + response.headers()["content-type"] + .to_str() + .unwrap() + .starts_with("text/plain") + ); + assert!( + response + .text() + .await + .unwrap() + .contains("rush_ingest_replay_batches_total") + ); + assert_eq!( + client + .get(format!("http://{address}/api/v1/tenants")) + .send() + .await + .unwrap() + .status(), + 404 + ); + shutdown.finish(); + tokio::time::timeout(std::time::Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); + } +} From 95584de5f9ce3a95e2489f87f5893af6e3f02e8c Mon Sep 17 00:00:00 2001 From: Mike Zupan Date: Sun, 15 Nov 2026 14:03:30 -0800 Subject: [PATCH 4/5] Measure export body completion errors and cancellation --- src/handlers/export.rs | 96 ++++++++++++++++ src/handlers/logs.rs | 15 ++- src/handlers/query.rs | 15 ++- src/main.rs | 38 +++++- src/query_governor.rs | 254 ++++++++++++++++++++++++++++++++++++++--- 5 files changed, 392 insertions(+), 26 deletions(-) diff --git a/src/handlers/export.rs b/src/handlers/export.rs index f876905..91d9e0e 100644 --- a/src/handlers/export.rs +++ b/src/handlers/export.rs @@ -876,6 +876,36 @@ enum StreamKind { Json, } +const EXPORT_ROWS_BUCKETS: &[f64] = &[ + 0.0, + 1.0, + 10.0, + 100.0, + 1_000.0, + 10_000.0, + 50_000.0, + 100_000.0, + 500_000.0, + 1_000_000.0, +]; + +struct SerializedRowsObservation { + metrics: Arc, + format: &'static str, + rows: u64, + outcome: &'static str, +} +impl Drop for SerializedRowsObservation { + fn drop(&mut self) { + self.metrics.observe_histogram_with( + "rush_export_serialized_rows", + &[("format", self.format), ("outcome", self.outcome)], + self.rows as f64, + EXPORT_ROWS_BUCKETS, + ); + } +} + /// State for both CSV and JSON streaming. Only a single decoded row and its /// serialized chunk are live at a time, so peak API memory is independent of /// the requested row count. @@ -888,6 +918,7 @@ struct ExportStreamState { max_bytes: u64, progress: ExportProgress, on_progress: Option, + metrics: SerializedRowsObservation, done: bool, } @@ -903,6 +934,11 @@ impl ExportStreamState { return None; } self.done = true; + self.metrics.outcome = if self.progress.truncated { + "truncated" + } else { + "complete" + }; let suffix = match self.kind { StreamKind::Csv if self.progress.truncated => { b"# truncated: export byte limit reached\n".to_vec() @@ -928,6 +964,7 @@ impl ExportStreamState { } } +#[allow(clippy::too_many_arguments)] fn stream_response( cursor: RowCursor, prelude: Vec, @@ -937,6 +974,7 @@ fn stream_response( max_rows: u64, max_bytes: u64, on_progress: Option, + metrics: Arc, ) -> Response where T: clickhouse::RowOwned + clickhouse::RowRead + Send + 'static, @@ -950,6 +988,15 @@ where max_bytes, progress: ExportProgress::default(), on_progress, + metrics: SerializedRowsObservation { + metrics, + format: match kind { + StreamKind::Csv => "csv", + StreamKind::Json => "json", + }, + rows: 0, + outcome: "cancelled", + }, done: false, }; @@ -960,6 +1007,7 @@ where if let Some(prelude) = state.prelude.take() { if prelude.len() as u64 + STREAM_SUFFIX_RESERVE > state.max_bytes { state.done = true; + state.metrics.outcome = "metadata_limit"; return Some(( Err::(std::io::Error::other( "export metadata exceeds the byte limit", @@ -981,6 +1029,7 @@ where Err(_error) => { tracing::error!(reason = "row_serialization", "export stream failed"); state.done = true; + state.metrics.outcome = "serialization_error"; return Some((Err(std::io::Error::other("export stream failed")), state)); } }; @@ -998,6 +1047,7 @@ where return state.finish_chunk().map(|suffix| (Ok(suffix), state)); } state.progress.rows += 1; + state.metrics.rows = state.progress.rows; state.progress.bytes += chunk.len() as u64; state.notify(); Some((Ok(Bytes::from(chunk)), state)) @@ -1006,6 +1056,7 @@ where Err(_error) => { tracing::error!(reason = "cursor_read", "export stream failed"); state.done = true; + state.metrics.outcome = "cursor_error"; Some((Err(std::io::Error::other("export stream failed")), state)) } } @@ -1043,6 +1094,7 @@ where /// Errors mid-stream terminate the body with an `io::Error`; axum surfaces that as a /// truncated/aborted response. The initial query has already executed by the time the /// first row is pulled, so query-level failures still abort the download cleanly. +#[allow(clippy::too_many_arguments)] pub fn stream_csv_response( cursor: RowCursor, prelude: String, @@ -1051,6 +1103,7 @@ pub fn stream_csv_response( max_rows: u64, max_bytes: u64, on_progress: Option, + metrics: Arc, ) -> Response where T: clickhouse::RowOwned + clickhouse::RowRead + Send + 'static, @@ -1064,6 +1117,7 @@ where max_rows, max_bytes, on_progress, + metrics, ) } @@ -1083,6 +1137,7 @@ pub fn json_query_preamble(query: serde_json::Value) -> Vec { bytes } +#[allow(clippy::too_many_arguments)] pub fn stream_json_response( cursor: RowCursor, prelude: Vec, @@ -1090,6 +1145,7 @@ pub fn stream_json_response( max_rows: u64, max_bytes: u64, on_progress: Option, + metrics: Arc, ) -> Response where T: clickhouse::RowOwned + clickhouse::RowRead + serde::Serialize + Send + 'static, @@ -1103,6 +1159,7 @@ where max_rows, max_bytes, on_progress, + metrics, ) } @@ -1110,6 +1167,45 @@ where mod tests { use super::*; + #[tokio::test] + async fn serialized_row_metrics_follow_actual_cursor_output() { + #[derive(clickhouse::Row, serde::Deserialize, serde::Serialize)] + struct TestRow { + value: u64, + } + let mock = clickhouse::test::Mock::new(); + mock.add(clickhouse::test::handlers::provide([ + TestRow { value: 1 }, + TestRow { value: 2 }, + ])); + let ch = clickhouse::Client::default() + .with_url(mock.url()) + .with_validation(false); + let cursor = ch.query("SELECT value").fetch::().unwrap(); + let metrics = Arc::new(crate::self_metrics::SelfMetrics::new()); + let response = stream_json_response( + cursor, + json_query_preamble(serde_json::json!({})), + "test.json", + 10, + 1024 * 1024, + None, + metrics.clone(), + ); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + assert_eq!( + serde_json::from_slice::(&body).unwrap()["count"], + 2 + ); + assert!( + metrics.render_prometheus().contains( + "rush_export_serialized_rows_sum{format=\"json\",outcome=\"complete\"} 2" + ) + ); + } + #[test] fn async_threshold_applies_to_every_large_effective_request() { assert!(!requires_async(0, sync_max_rows())); diff --git a/src/handlers/logs.rs b/src/handlers/logs.rs index ac0c9bc..21e6811 100644 --- a/src/handlers/logs.rs +++ b/src/handlers/logs.rs @@ -1025,6 +1025,8 @@ pub async fn export_logs( } }; let budget = guard.budget().clone(); + let deadline = tokio::time::Instant::now() + + std::time::Duration::from_secs(budget.request_timeout_secs); let progress = background_state .export_jobs .progress_callback(&background_job_id); @@ -1049,6 +1051,7 @@ pub async fn export_logs( limit, max_bytes, Some(progress), + background_state.self_metrics.clone(), ), export::ExportFormat::Json => export::stream_json_response( cursor, @@ -1057,8 +1060,12 @@ pub async fn export_logs( limit, max_bytes, Some(progress), + background_state.self_metrics.clone(), ), }; + let response = crate::query_governor::retain_artifact_until_body_deadline( + response, guard, deadline, + ); background_state .export_jobs .write_response(&background_job_id, response) @@ -1066,13 +1073,9 @@ pub async fn export_logs( }; let result = crate::query_governor::with_budget( budget.clone(), - tokio::time::timeout( - std::time::Duration::from_secs(budget.request_timeout_secs), - work, - ), + tokio::time::timeout_at(deadline, work), ) .await; - drop(guard); match result { Ok(Ok(())) => {} _ if background_state @@ -1138,6 +1141,7 @@ pub async fn export_logs( limit, max_bytes, None, + state.self_metrics.clone(), )) } export::ExportFormat::Json => { @@ -1168,6 +1172,7 @@ pub async fn export_logs( limit, max_bytes, None, + state.self_metrics.clone(), )) } } diff --git a/src/handlers/query.rs b/src/handlers/query.rs index 5a5dcac..3a62a2b 100644 --- a/src/handlers/query.rs +++ b/src/handlers/query.rs @@ -420,6 +420,8 @@ pub async fn export_query( } }; let budget = guard.budget().clone(); + let deadline = tokio::time::Instant::now() + + std::time::Duration::from_secs(budget.request_timeout_secs); let progress = background_state .export_jobs .progress_callback(&background_job_id); @@ -453,6 +455,7 @@ pub async fn export_query( limit, max_bytes, Some(progress), + background_state.self_metrics.clone(), ), export::ExportFormat::Json => export::stream_json_response( cursor, @@ -461,8 +464,12 @@ pub async fn export_query( limit, max_bytes, Some(progress), + background_state.self_metrics.clone(), ), }; + let response = crate::query_governor::retain_artifact_until_body_deadline( + response, guard, deadline, + ); background_state .export_jobs .write_response(&background_job_id, response) @@ -470,13 +477,9 @@ pub async fn export_query( }; let result = crate::query_governor::with_budget( budget.clone(), - tokio::time::timeout( - std::time::Duration::from_secs(budget.request_timeout_secs), - work, - ), + tokio::time::timeout_at(deadline, work), ) .await; - drop(guard); match result { Ok(Ok(())) => {} _ if background_state @@ -547,6 +550,7 @@ pub async fn export_query( limit, max_bytes, None, + state.self_metrics.clone(), )) } export::ExportFormat::Json => { @@ -577,6 +581,7 @@ pub async fn export_query( limit, max_bytes, None, + state.self_metrics.clone(), )) } } diff --git a/src/main.rs b/src/main.rs index b1e60cd..9f25c5f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -434,7 +434,7 @@ async fn http_metrics_middleware( return next.run(req).await; } - let method = req.method().as_str().to_string(); + let method = http_method_label(req.method()).to_string(); let sm = state.self_metrics.clone(); // In-flight gauge: inc on entry, dec on exit (even on panic-free early returns). @@ -468,9 +468,29 @@ async fn http_metrics_middleware( elapsed_ms, ); + sm.observe_histogram( + "rush_http_response_headers_duration_ms", + &[("route", route.as_str()), ("method", method.as_str())], + elapsed_ms, + ); resp } +fn http_method_label(method: &axum::http::Method) -> &'static str { + match method.as_str() { + "GET" => "GET", + "HEAD" => "HEAD", + "POST" => "POST", + "PUT" => "PUT", + "DELETE" => "DELETE", + "CONNECT" => "CONNECT", + "OPTIONS" => "OPTIONS", + "TRACE" => "TRACE", + "PATCH" => "PATCH", + _ => "OTHER", + } +} + /// Open `GET /metrics` handler: renders the self-metrics registry as Prometheus text /// exposition (version 0.0.4). No auth — same posture as `/healthz`. async fn metrics_handler(State(state): State) -> Response { @@ -1452,6 +1472,11 @@ where .await; match result { Ok((response, outcome)) => { + let outcome = if class == WorkloadClass::Export && outcome == "completed" { + "response_ready" + } else { + outcome + }; metrics.inc_counter( "rush_query_requests_total", &[("workload", label), ("outcome", outcome)], @@ -4470,6 +4495,17 @@ mod tenant_auth_tests { use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; + #[test] + fn arbitrary_http_methods_do_not_create_unbounded_metric_labels() { + assert_eq!(super::http_method_label(&axum::http::Method::GET), "GET"); + assert_eq!( + super::http_method_label( + &axum::http::Method::from_bytes(b"CUSTOM-TENANT-SECRET").unwrap() + ), + "OTHER" + ); + } + #[test] fn startup_validates_api_key_secret_before_running_migrations() { let source = include_str!("main.rs"); diff --git a/src/query_governor.rs b/src/query_governor.rs index 0a1207d..8438b3a 100644 --- a/src/query_governor.rs +++ b/src/query_governor.rs @@ -582,6 +582,73 @@ pub fn retain_admission_until_body_end( retain_admission_until_body_deadline(response, guard, deadline) } +const EXPORT_BODY_LATENCY_BUCKETS_MS: &[f64] = &[ + 10.0, 50.0, 100.0, 500.0, 1_000.0, 5_000.0, 10_000.0, 30_000.0, 60_000.0, 120_000.0, 300_000.0, + 600_000.0, +]; +const EXPORT_BODY_BYTES_BUCKETS: &[f64] = &[ + 0.0, + 1_024.0, + 16_384.0, + 262_144.0, + 1_048_576.0, + 4_194_304.0, + 16_777_216.0, + 67_108_864.0, + 268_435_456.0, + 1_073_741_824.0, + 4_294_967_296.0, +]; + +struct ExportBodyObservation { + metrics: Arc, + kind: &'static str, + started: std::time::Instant, + bytes: u64, + recorded: bool, +} + +impl ExportBodyObservation { + fn new(metrics: Arc, kind: &'static str) -> Self { + metrics.add_gauge("rush_export_streams_inflight", &[("kind", kind)], 1.0); + Self { + metrics, + kind, + started: std::time::Instant::now(), + bytes: 0, + recorded: false, + } + } + fn finish(&mut self, outcome: &'static str) { + if self.recorded { + return; + } + self.recorded = true; + self.metrics + .add_gauge("rush_export_streams_inflight", &[("kind", self.kind)], -1.0); + let labels = [("kind", self.kind), ("outcome", outcome)]; + self.metrics + .inc_counter("rush_export_streams_total", &labels, 1); + self.metrics.observe_histogram_with( + "rush_export_stream_duration_ms", + &labels, + self.started.elapsed().as_secs_f64() * 1000.0, + EXPORT_BODY_LATENCY_BUCKETS_MS, + ); + self.metrics.observe_histogram_with( + "rush_export_stream_bytes", + &labels, + self.bytes as f64, + EXPORT_BODY_BYTES_BUCKETS, + ); + } +} +impl Drop for ExportBodyObservation { + fn drop(&mut self) { + self.finish("cancelled"); + } +} + /// Use the request's original deadline so body upload and handler execution do /// not grant a fresh timeout when an export starts streaming. pub fn retain_admission_until_body_deadline( @@ -589,58 +656,92 @@ pub fn retain_admission_until_body_deadline( guard: QueryAdmissionGuard, deadline: tokio::time::Instant, ) -> axum::response::Response { + observe_export_body(response, guard, deadline, "response") +} + +/// Background artifact generation has the same lifecycle but a separate kind, +/// so its bytes are not confused with a later HTTP artifact download. +pub fn retain_artifact_until_body_deadline( + response: axum::response::Response, + guard: QueryAdmissionGuard, + deadline: tokio::time::Instant, +) -> axum::response::Response { + observe_export_body(response, guard, deadline, "artifact") +} + +fn observe_export_body( + response: axum::response::Response, + guard: QueryAdmissionGuard, + deadline: tokio::time::Instant, + kind: &'static str, +) -> axum::response::Response { + let observation = ExportBodyObservation::new(guard.metrics.clone(), kind); let (parts, body) = response.into_parts(); let state = ( - Box::pin(body.into_data_stream()), + Some(Box::pin(body.into_data_stream())), Some(guard), deadline, false, + observation, ); - let stream = - futures_util::stream::unfold(state, |(mut body, mut guard, deadline, done)| async move { + let stream = futures_util::stream::unfold( + state, + |(mut body, mut guard, deadline, done, mut observation)| async move { if done { return None; } - // timeout_at polls the future first. Explicitly enforce an expired - // deadline even if the next chunk is immediately ready. if tokio::time::Instant::now() >= deadline { + observation.finish("timeout"); + body.take(); guard.take(); return Some(( Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "export response exceeded its workload timeout", )), - (body, guard, deadline, true), + (body, guard, deadline, true, observation), )); } - match tokio::time::timeout_at(deadline, body.next()).await { - Ok(Some(Ok(bytes))) => Some(( - Ok::(bytes), - (body, guard, deadline, false), - )), + let next = + tokio::time::timeout_at(deadline, body.as_mut().expect("active body").next()).await; + match next { + Ok(Some(Ok(bytes))) => { + observation.bytes = observation.bytes.saturating_add(bytes.len() as u64); + Some(( + Ok::(bytes), + (body, guard, deadline, false, observation), + )) + } Ok(Some(Err(error))) => { + observation.finish("body_error"); + body.take(); guard.take(); Some(( Err(std::io::Error::other(error)), - (body, guard, deadline, true), + (body, guard, deadline, true, observation), )) } Ok(None) => { + observation.finish("complete"); + body.take(); guard.take(); None } Err(_) => { + observation.finish("timeout"); + body.take(); guard.take(); Some(( Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "export response exceeded its workload timeout", )), - (body, guard, deadline, true), + (body, guard, deadline, true, observation), )) } } - }); + }, + ); axum::response::Response::from_parts(parts, axum::body::Body::from_stream(stream)) } @@ -916,13 +1017,131 @@ mod tests { assert!(dropped.load(Ordering::SeqCst)); } + #[tokio::test] + async fn stream_metrics_distinguish_eof_error_timeout_and_artifact_generation() { + let metrics = Arc::new(SelfMetrics::new()); + let governor = QueryGovernor::new(QueryGovernorConfig::default(), metrics.clone()).unwrap(); + let guard = governor + .admit(WorkloadClass::Export, "tenant-secret") + .await + .unwrap(); + let response = retain_admission_until_body_end( + axum::response::Response::new(axum::body::Body::from("hello")), + guard, + Duration::from_secs(1), + ); + assert!( + metrics + .render_prometheus() + .contains("rush_export_streams_inflight{kind=\"response\"} 1") + ); + assert!( + !metrics + .render_prometheus() + .contains("rush_export_streams_total") + ); + assert_eq!( + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap() + .len(), + 5 + ); + let guard = governor + .admit(WorkloadClass::Export, "tenant-secret") + .await + .unwrap(); + let body = axum::body::Body::from_stream(futures_util::stream::once(async { + Err::(std::io::Error::other("secret error")) + })); + assert!( + axum::body::to_bytes( + retain_admission_until_body_end( + axum::response::Response::new(body), + guard, + Duration::from_secs(1) + ) + .into_body(), + 1024 + ) + .await + .is_err() + ); + let guard = governor + .admit(WorkloadClass::Export, "tenant-secret") + .await + .unwrap(); + let response = retain_artifact_until_body_deadline( + axum::response::Response::new(axum::body::Body::from("artifact")), + guard, + tokio::time::Instant::now() + Duration::from_secs(1), + ); + axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(); + let text = metrics.render_prometheus(); + assert!( + text.contains("rush_export_streams_total{kind=\"response\",outcome=\"complete\"} 1") + ); + assert!( + text.contains("rush_export_streams_total{kind=\"response\",outcome=\"body_error\"} 1") + ); + assert!( + text.contains("rush_export_streams_total{kind=\"artifact\",outcome=\"complete\"} 1") + ); + assert!( + text.contains("rush_export_stream_bytes_sum{kind=\"response\",outcome=\"complete\"} 5") + ); + assert!(!text.contains("secret")); + } + + #[tokio::test] + async fn timed_out_stream_drops_cursor_even_if_consumer_keeps_failed_body() { + struct DropWatch(Arc); + impl Drop for DropWatch { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } + } + let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let watch = DropWatch(dropped.clone()); + let stream = futures_util::stream::pending::>().map( + move |chunk| { + let _ = &watch; + chunk + }, + ); + let metrics = Arc::new(SelfMetrics::new()); + let governor = QueryGovernor::new(QueryGovernorConfig::default(), metrics.clone()).unwrap(); + let guard = governor + .admit(WorkloadClass::Export, "tenant-a") + .await + .unwrap(); + let response = retain_admission_until_body_end( + axum::response::Response::new(axum::body::Body::from_stream(stream)), + guard, + Duration::from_millis(5), + ); + let mut body = response.into_body().into_data_stream(); + assert!(body.next().await.unwrap().is_err()); + assert!(dropped.load(Ordering::SeqCst)); + let text = metrics.render_prometheus(); + assert!(text.contains("rush_export_streams_inflight{kind=\"response\"} 0")); + assert!( + text.contains("rush_export_streams_total{kind=\"response\",outcome=\"timeout\"} 1") + ); + drop(body); + assert_eq!(text, metrics.render_prometheus()); + } + #[tokio::test] async fn streaming_response_holds_permit_until_body_is_dropped() { let mut config = QueryGovernorConfig::default(); config.export.global_concurrency = 1; config.export.per_tenant_concurrency = 1; config.export.queue_timeout_ms = 0; - let governor = QueryGovernor::new(config, Arc::new(SelfMetrics::new())).unwrap(); + let metrics = Arc::new(SelfMetrics::new()); + let governor = QueryGovernor::new(config, metrics.clone()).unwrap(); let guard = governor .admit(WorkloadClass::Export, "tenant-a") .await @@ -941,6 +1160,11 @@ mod tests { Err(AdmissionError::TenantBusy { .. }) )); drop(response); + let text = metrics.render_prometheus(); + assert!( + text.contains("rush_export_streams_total{kind=\"response\",outcome=\"cancelled\"} 1") + ); + assert!(text.contains("rush_export_streams_inflight{kind=\"response\"} 0")); assert!( governor .admit(WorkloadClass::Export, "tenant-a") From 2766e26ad776cab262a41e67ef067f4e0b2d16ae Mon Sep 17 00:00:00 2001 From: Mike Zupan Date: Sun, 15 Nov 2026 14:25:54 -0800 Subject: [PATCH 5/5] Accept restricted frontend health telemetry through authenticated sessions --- .github/workflows/security.yml | 3 + src/handlers/frontend_telemetry.rs | 567 ++++++++++++++++++ src/handlers/frontend_telemetry_tests.rs | 158 +++++ src/handlers/mod.rs | 2 + src/handlers/rum.rs | 14 +- src/main.rs | 63 ++ tests/fixtures/frontend-clickhouse.xml | 19 + .../fixtures/frontend-health-all-events.json | 112 ++++ tests/fixtures/frontend-health-sdk.json | 297 +++++++++ tests/frontend_telemetry_session.rs | 335 +++++++++++ tests/run-frontend-telemetry.sh | 22 + 11 files changed, 1590 insertions(+), 2 deletions(-) create mode 100644 src/handlers/frontend_telemetry.rs create mode 100644 src/handlers/frontend_telemetry_tests.rs create mode 100644 tests/fixtures/frontend-clickhouse.xml create mode 100644 tests/fixtures/frontend-health-all-events.json create mode 100644 tests/fixtures/frontend-health-sdk.json create mode 100644 tests/frontend_telemetry_session.rs create mode 100644 tests/run-frontend-telemetry.sh diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index ba7b8b6..a51279e 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -63,5 +63,8 @@ jobs: # PRs stay bounded; the weekly schedule exercises a much larger corpus. PROPTEST_CASES: ${{ github.event_name == 'schedule' && '4096' || '256' }} + - name: Test frontend telemetry session delivery + run: bash tests/run-frontend-telemetry.sh + - name: Build release targets run: cargo build --release --all-features diff --git a/src/handlers/frontend_telemetry.rs b/src/handlers/frontend_telemetry.rs new file mode 100644 index 0000000..ec815a5 --- /dev/null +++ b/src/handlers/frontend_telemetry.rs @@ -0,0 +1,567 @@ +//! Session-only operational telemetry for the Rush frontend, never general ingestion. +use super::rum::{RumIngestEvent, RumIngestPayload}; +use crate::{AppState, RequestIdentity, TenantContext}; +use axum::{ + Extension, + extract::State, + http::{HeaderMap, StatusCode, header}, + response::{IntoResponse, Response}, +}; +use serde_json::{Map, Value}; +use std::time::{Duration, Instant}; + +pub const MAX_BODY_BYTES: usize = 64 * 1024; +const MAX_EVENTS: usize = 1000; +const USER_REQUESTS_PER_MINUTE: u64 = 120; +const TENANT_REQUESTS_PER_MINUTE: u64 = 1200; +const MAX_COUNTER: f64 = 9_007_199_254_740_991.0; +const MAX_DURATION_MS: f64 = 86_400_000.0; +type Rejection = (StatusCode, String); +fn denied(status: StatusCode, message: &str) -> Rejection { + (status, message.into()) +} + +pub async fn ingest( + State(state): State, + Extension(tenant): Extension, + Extension(identity): Extension, + headers: HeaderMap, + body: axum::body::Bytes, +) -> Result { + if !session_identity(&identity, &tenant.tenant_id) { + return Err(denied( + StatusCode::UNAUTHORIZED, + "frontend telemetry requires a browser session", + )); + } + // Use the same request-scoped session validation as tenant middleware. Neither + // a query key nor an unverified header may turn into a frontend ingest grant. + let caller = super::users::require_auth(&state, &headers).await?; + let session_tenant = state + .config_db + .resolve_session_tenant(&caller.0, &caller.4, &caller.3, None) + .await + .map_err(|_| { + denied( + StatusCode::SERVICE_UNAVAILABLE, + "session tenant unavailable", + ) + })? + .ok_or_else(|| denied(StatusCode::FORBIDDEN, "session tenant unavailable"))?; + if session_tenant != tenant.tenant_id || caller.0 != identity.actor_id { + return Err(denied( + StatusCode::FORBIDDEN, + "frontend telemetry cannot change the session tenant", + )); + } + if let Err(retry_after) = consume_rate_limits( + &state.ingest_key_limiter, + &identity.actor_id, + &tenant.tenant_id, + Instant::now(), + ) { + return Ok(( + StatusCode::TOO_MANY_REQUESTS, + [(header::RETRY_AFTER, retry_after.to_string())], + "frontend telemetry rate limit exceeded", + ) + .into_response()); + } + state.ingest_limits.check_body("rum", &body)?; + state.ingest_limits.check_decompressed("rum", body.len())?; + if body.len() > MAX_BODY_BYTES { + return Err(denied( + StatusCode::PAYLOAD_TOO_LARGE, + "frontend telemetry payload too large", + )); + } + if !super::rum::rum_enabled(&state).await { + return Err(denied(StatusCode::FORBIDDEN, "RUM ingestion is disabled")); + } + let payload: RumIngestPayload = serde_json::from_slice(&body).map_err(|_| { + denied( + StatusCode::BAD_REQUEST, + "invalid frontend telemetry payload", + ) + })?; + state + .ingest_limits + .check_entities("rum", payload.events.len())?; + let payload = sanitize_payload(payload)?; + if payload.events.is_empty() { + return Ok(axum::Json(serde_json::json!({"accepted":0})).into_response()); + } + // Existing tenant signal gate, durable writer/backpressure, usage metering, + // and finite-label RUM ingest metrics remain the sole write path. + match super::rum::ingest_events(state, tenant, payload, body.len()).await { + Ok(response) => Ok(response.into_response()), + Err((status, message)) + if matches!( + status, + StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE + ) => + { + Ok((status, [(header::RETRY_AFTER, "10")], message).into_response()) + } + Err(error) => Err(error), + } +} +fn session_identity(identity: &RequestIdentity, tenant: &str) -> bool { + identity.authenticated + && identity.credential_type == "session" + && identity.actor_type == "user" + && !identity.actor_id.is_empty() + && identity.tenant_id == tenant +} +fn consume_rate_limits( + limiter: &dashmap::DashMap, + user: &str, + tenant: &str, + now: Instant, +) -> Result<(), u64> { + for (key, limit) in [ + ( + format!("frontend-user:{tenant}:{user}"), + USER_REQUESTS_PER_MINUTE, + ), + ( + format!("frontend-tenant:{tenant}"), + TENANT_REQUESTS_PER_MINUTE, + ), + ] { + if limiter.len() >= 100_000 && !limiter.contains_key(&key) { + limiter.retain(|_, (_, started)| { + now.saturating_duration_since(*started) < Duration::from_secs(60) + }); + if limiter.len() >= 100_000 { + return Err(60); + } + } + let mut entry = limiter.entry(key).or_insert((0, now)); + let elapsed = now.saturating_duration_since(entry.1); + if elapsed >= Duration::from_secs(60) { + *entry = (0, now); + } + if entry.0 >= limit { + return Err(60 - elapsed.as_secs().min(59)); + } + entry.0 += 1; + } + Ok(()) +} +fn safe_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= 64 + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)) +} +fn category_path(path: &str) -> String { + let category = path + .split(['?', '#']) + .next() + .unwrap_or("") + .split('/') + .nth(1) + .unwrap_or(""); + match category { + "" => "/", + "apm" | "logs" | "metrics" | "rum" | "dashboards" | "notebooks" | "anomalies" | "slos" + | "monitors" | "profiles" | "services" | "investigate" | "control" + | "telemetry-quality" | "noise-waste" | "capacity" => return format!("/{category}"), + _ => "/other", + } + .into() +} +fn sanitize_payload(mut payload: RumIngestPayload) -> Result { + if payload.events.len() > MAX_EVENTS { + return Err(denied( + StatusCode::PAYLOAD_TOO_LARGE, + "frontend telemetry contains too many events", + )); + } + let meta = &mut payload.meta; + if !safe_name(&meta.app_name) { + return Err(denied( + StatusCode::BAD_REQUEST, + "invalid frontend application name", + )); + } + if meta.app_name != "rush-ui" { + meta.app_name = format!("rush-ui:{}", meta.app_name); + } + if meta.app_version.is_empty() + || meta.app_version.len() > 64 + || !meta + .app_version + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-+".contains(&b)) + { + meta.app_version = "unknown".into(); + } + if !matches!( + meta.environment.as_str(), + "production" | "development" | "test" | "staging" + ) { + meta.environment = "unknown".into(); + } + meta.page_path = category_path(&meta.page_path); + meta.session_id.clear(); + meta.user_id.clear(); + meta.page_url.clear(); + meta.referrer.clear(); + meta.view_name.clear(); + // These optional metadata strings are not needed for operational health. + meta.browser_name.clear(); + meta.browser_version.clear(); + meta.os_name.clear(); + meta.os_version.clear(); + if !matches!(meta.device_type.as_str(), "desktop" | "mobile" | "tablet") { + meta.device_type = "unknown".into(); + } + meta.screen_width = meta.screen_width.min(16384); + meta.screen_height = meta.screen_height.min(16384); + payload.events = payload + .events + .into_iter() + .filter_map(sanitize_event) + .collect(); + Ok(payload) +} +fn sanitize_event(mut event: RumIngestEvent) -> Option { + match event.event_type.as_str() { + "custom" + if matches!( + event.event_name.as_str(), + "frontend_started" + | "query_request_cache" + | "polling_refresh" + | "frontend_api_request" + | "frontend_api_attempt" + | "frontend_api_retry" + | "frontend_api_body" + | "frontend_api_probe" + | "frontend_api_outage" + | "frontend_exception" + | "frontend_navigation" + | "frontend_investigation_stream" + | "frontend_investigation_first_response" + | "frontend_investigation_discarded" + | "frontend_telemetry_dropped" + ) => {} + "web_vital" + if matches!( + event.vital_name.as_str(), + "LCP" | "CLS" | "INP" | "FCP" | "TTFB" + ) => + { + event.event_name.clear(); + } + "pageview" => { + event.event_name = "pageview".into(); + } + "error" | "frustration" | "interaction" | "long_task" | "loaf" => { + event.event_name.clear(); + } + _ => return None, + } + if event.event_type != "web_vital" { + event.vital_name.clear(); + event.vital_rating.clear(); + event.vital_value = 0.0; + } + event.timestamp = None; + event.trace_id.clear(); + event.span_id.clear(); + event.duration_ms = event.duration_ms.clamp(0.0, MAX_DURATION_MS); + event.vital_value = event.vital_value.clamp(0.0, MAX_DURATION_MS); + if !matches!( + event.vital_rating.as_str(), + "good" | "needs-improvement" | "poor" + ) { + event.vital_rating.clear(); + } + if event.event_type == "error" { + event.error_message = "Client error".into(); + if !matches!( + event.error_type.as_str(), + "Error" + | "TypeError" + | "RangeError" + | "ReferenceError" + | "SyntaxError" + | "URIError" + | "EvalError" + | "UnhandledRejection" + ) { + event.error_type = "Error".into(); + } + } else { + event.error_message.clear(); + event.error_type.clear(); + } + event.error_stack.clear(); + let tag = event + .interaction_target + .split(|c: char| !c.is_ascii_alphabetic()) + .next() + .unwrap_or("") + .to_ascii_lowercase(); + event.interaction_target = match tag.as_str() { + "button" | "a" | "input" | "textarea" | "select" | "canvas" | "div" | "span" | "label" + | "form" => tag, + _ => String::new(), + }; + if !matches!( + event.interaction_type.as_str(), + "click" | "rage_click" | "dead_click" | "error_click" + ) { + event.interaction_type.clear(); + } + event.attributes = sanitize_attributes(&event.attributes); + Some(event) +} +fn sanitize_attributes(raw: &str) -> String { + let Ok(Value::Object(attributes)) = serde_json::from_str::(raw) else { + return "{}".into(); + }; + let mut safe = Map::new(); + for (key, value) in attributes { + if numeric_attribute(&key) { + if let Some(number) = value.as_f64().filter(|n| n.is_finite() && *n >= 0.0) { + safe.insert( + key, + if number >= MAX_COUNTER { + serde_json::json!(9_007_199_254_740_991_u64) + } else { + value + }, + ); + } + } else if let Some(allowed) = string_attribute(&key) + && let Some(value) = value.as_str() + { + safe.insert( + key, + Value::String( + if allowed.contains(&value) { + value + } else { + "other" + } + .into(), + ), + ); + } + } + Value::Object(safe).to_string() +} +fn numeric_attribute(key: &str) -> bool { + matches!( + key, + "count" + | "duration_count" + | "duration_ms_sum" + | "duration_ms_max" + | "startup_duration_ms" + | "cache_hits" + | "fresh_hits" + | "stale_hits" + | "revalidation_failures" + | "oversized_bypasses" + | "cache_entries" + | "cache_bytes" + | "in_flight" + | "in_flight_deduplications" + | "superseded_or_aborted" + | "cache_misses" + | "cache_evictions" + | "refreshes" + | "failures" + | "skipped_overlaps" + | "backoffs" + | "average_duration_ms" + | "maximum_duration_ms" + | "maximum_backoff_ms" + | "start" + | "blockingDuration" + | "scripts" + ) || matches!( + key, + "duration_ms_le_50" + | "duration_ms_le_100" + | "duration_ms_le_250" + | "duration_ms_le_500" + | "duration_ms_le_1000" + | "duration_ms_le_2500" + | "duration_ms_le_5000" + | "duration_ms_le_10000" + | "duration_ms_le_30000" + | "duration_ms_le_60000" + ) +} +fn string_attribute(key: &str) -> Option<&'static [&'static str]> { + Some(match key { + "cancel_reason" => &[ + "caller", + "none", + "session_changed", + "superseded_or_invalidated", + ], + "category" => &[ + "alert_list", + "anomaly", + "capacity", + "dashboard", + "explore_live", + "integration_activity", + "notebook", + "service_graph", + "slo_detail", + "slo_list", + ], + "kind" => &["evidence", "report"], + "method" => &[ + "DELETE", "GET", "HEAD", "OPTIONS", "OTHER", "PATCH", "POST", "PUT", + ], + "operation" => &[ + "anomalies", + "anomaly-events", + "apikeys", + "auth", + "bubbleup", + "dashboards", + "explore", + "export", + "exports", + "features", + "frontend", + "groups", + "integrations", + "investigate", + "license", + "logs", + "monitors", + "notebooks", + "other", + "profiles", + "prometheus", + "query", + "retention", + "roles", + "rum", + "services", + "sessions", + "settings", + "slo", + "slos", + "stats", + "suggest", + "system", + "tenants", + "traces", + "users", + ], + "outcome" => &[ + "agent_error", + "body_error", + "cancelled", + "completed", + "connection_error", + "decode_error", + "duplicated", + "failed", + "http_5xx", + "http_error", + "idle_timeout", + "interrupted", + "invalid_response", + "limit_exceeded", + "network", + "network_error", + "offline", + "other_error", + "reachable", + "redirected", + "stream_error", + "success", + "timeout", + ], + "reason" => &[ + "event_limit", + "field_truncated", + "http_5xx", + "http_error", + "invalid_response", + "malformed", + "network", + "offline", + "oversized", + "timeout", + "unknown_event", + ], + "route" => &[ + "alert-channel-add", + "alerts", + "anomaly", + "anomaly-add", + "anomaly-detail", + "argocd-detail", + "audit", + "capacity", + "dashboard", + "dashboards", + "deploys", + "explore", + "fluxcd-detail", + "funnels", + "integration-page", + "integrations", + "investigate", + "kubernetes-detail", + "log-views", + "login", + "logs", + "metric-detail", + "metrics", + "monitor-create", + "monitor-detail", + "monitor-edit", + "noise-waste", + "notebook", + "notebooks", + "other", + "profiles", + "rum", + "rum-detail", + "service-detail", + "services", + "settings", + "slo-detail", + "slos", + "sre-agent", + "sre-agent-session", + "sso-setup", + "telemetry-quality", + "trace", + "traces", + "usage", + ], + "source" => &["bootstrap", "router", "vue"], + "stage" => &[ + "event", + "lazy_chunk", + "mount", + "other", + "render", + "setup", + "watcher", + ], + "status_class" => &["1xx", "2xx", "3xx", "4xx", "5xx", "none"], + _ => return None, + }) +} + +#[cfg(test)] +#[path = "frontend_telemetry_tests.rs"] +mod tests; diff --git a/src/handlers/frontend_telemetry_tests.rs b/src/handlers/frontend_telemetry_tests.rs new file mode 100644 index 0000000..15b9761 --- /dev/null +++ b/src/handlers/frontend_telemetry_tests.rs @@ -0,0 +1,158 @@ +use super::*; +use serde_json::json; +fn payload(events: Value) -> RumIngestPayload { + serde_json::from_value(json!({"meta":{"app_name":"rush-ui","app_version":"0.1.13","environment":"production","session_id":"sensitive-session","user_id":"sensitive-user","page_url":"https://rush.example/logs?secret=sensitive-query","page_path":"/logs/sensitive-id?secret=sensitive-query","view_name":"sensitive-title","referrer":"sensitive-referrer"},"events":events})).unwrap() +} +#[test] +fn privacy_is_enforced_without_losing_bounded_health_signals() { + let input = payload(json!([ + {"event_type":"custom","event_name":"frontend_api_request","timestamp":123,"trace_id":"sensitive-trace","span_id":"sensitive-span","vital_name":"sensitive-vital","attributes":json!({"operation":"query","method":"GET","outcome":"decode_error","count":2,"duration_ms_sum":12.5,"status_class":"2xx","query":"sensitive-query","password":"sensitive-password","route":"sensitive-route"}).to_string()}, + {"event_type":"pageview","event_name":"sensitive-title","trace_id":"sensitive-trace","span_id":"sensitive-span"}, + {"event_type":"error","error_message":"sensitive-password","error_stack":"sensitive-stack","error_type":"sensitive-type"}, + {"event_type":"custom","event_name":"arbitrary-log-relay","attributes":"sensitive-payload"}, + {"event_type":"resource","event_name":"https://sensitive-url/"} + ])); + let output = sanitize_payload(input).unwrap(); + assert_eq!(output.events.len(), 3); + assert_eq!(output.meta.page_path, "/logs"); + let encoded = serde_json::to_string(&output).unwrap(); + assert!(!encoded.contains("sensitive")); + assert!(!encoded.contains("arbitrary-log-relay")); + assert_eq!(output.events[0].timestamp, None); + let attrs: Value = serde_json::from_str(&output.events[0].attributes).unwrap(); + assert_eq!(attrs["operation"], "query"); + assert_eq!(attrs["count"].as_f64(), Some(2.0)); + assert_eq!(attrs["duration_ms_sum"].as_f64(), Some(12.5)); + assert_eq!(attrs["route"], "other"); + assert_eq!(output.events[1].event_name, "pageview"); + assert!( + output + .events + .iter() + .all(|e| e.trace_id.is_empty() && e.span_id.is_empty()) + ); + assert_eq!(output.events[2].error_message, "Client error"); + assert_eq!(output.events[2].error_type, "Error"); +} +#[test] +fn real_sdk_fixture_preserves_the_allowed_event_and_attribute_contract() { + let mut payloads: Vec = serde_json::from_str(include_str!( + "../../tests/fixtures/frontend-health-sdk.json" + )) + .unwrap(); + payloads.push( + serde_json::from_str(include_str!( + "../../tests/fixtures/frontend-health-all-events.json" + )) + .unwrap(), + ); + let mut accepted = Vec::new(); + for original in payloads { + let events = original + .events + .iter() + .map(|event| { + ( + event.event_type.clone(), + event.event_name.clone(), + event.attributes.clone(), + ) + }) + .collect::>(); + let sanitized = sanitize_payload(original).unwrap(); + assert_eq!(sanitized.events.len(), events.len()); + for (event, (kind, name, attrs)) in sanitized.events.iter().zip(events) { + assert_eq!(event.event_type, kind); + if kind == "custom" { + assert_eq!(event.event_name, name); + let before: Value = serde_json::from_str(&attrs).unwrap(); + let after: Value = serde_json::from_str(&event.attributes).unwrap(); + for (key, value) in before.as_object().unwrap() { + let retained = &after[key]; + if value.is_number() { + assert_eq!(retained.as_f64(), value.as_f64(), "{name}.{key}"); + } else { + assert_eq!(retained, value, "{name}.{key}"); + } + } + } + } + accepted.push(sanitized); + } + if let Ok(path) = std::env::var("RUSH_FRONTEND_SANITIZED_FIXTURE_OUTPUT") { + std::fs::write(path, serde_json::to_vec_pretty(&accepted).unwrap()).unwrap(); + } +} +#[test] +fn identity_checks_accept_only_a_validated_same_tenant_session() { + let identity = RequestIdentity { + tenant_id: "tenant-a".into(), + authenticated: true, + actor_id: "user-1".into(), + actor_name: "viewer".into(), + actor_type: "user".into(), + credential_type: "session".into(), + }; + assert!(session_identity(&identity, "tenant-a")); + assert!(!session_identity(&identity, "tenant-b")); + for kind in ["anonymous", "query_key", "ingest_key", "internal_sre"] { + let mut changed = identity.clone(); + changed.credential_type = kind.into(); + assert!(!session_identity(&changed, "tenant-a")); + } + let mut changed = identity.clone(); + changed.authenticated = false; + assert!(!session_identity(&changed, "tenant-a")); + changed = identity; + changed.actor_id.clear(); + assert!(!session_identity(&changed, "tenant-a")); +} +#[test] +fn user_and_tenant_limits_are_independent_and_reset() { + let limiter = dashmap::DashMap::new(); + let now = Instant::now(); + for _ in 0..USER_REQUESTS_PER_MINUTE { + consume_rate_limits(&limiter, "u1", "t1", now).unwrap(); + } + assert_eq!(consume_rate_limits(&limiter, "u1", "t1", now), Err(60)); + consume_rate_limits(&limiter, "u2", "t1", now).unwrap(); + consume_rate_limits(&limiter, "u1", "t1", now + Duration::from_secs(60)).unwrap(); + let limiter = dashmap::DashMap::new(); + for index in 0..TENANT_REQUESTS_PER_MINUTE { + consume_rate_limits(&limiter, &format!("u{index}"), "t1", now).unwrap(); + } + assert!(consume_rate_limits(&limiter, "another", "t1", now).is_err()); + consume_rate_limits(&limiter, "another", "t2", now).unwrap(); +} +#[test] +fn payload_and_attribute_values_remain_bounded() { + let event = json!({"event_type":"custom","event_name":"frontend_started","duration_ms":-1,"attributes":json!({"startup_duration_ms":1e30,"count":-1,"method":"sensitive-url","arbitrary":"sensitive-token"}).to_string()}); + let result = sanitize_payload(payload(json!([event.clone()]))).unwrap(); + let attrs: Value = serde_json::from_str(&result.events[0].attributes).unwrap(); + assert_eq!(attrs["startup_duration_ms"].as_f64(), Some(MAX_COUNTER)); + assert!(attrs.get("count").is_none()); + assert_eq!(attrs["method"], "other"); + assert!(attrs.get("arbitrary").is_none()); + let too_many = Value::Array(vec![event; MAX_EVENTS + 1]); + assert_eq!( + sanitize_payload(payload(too_many)).unwrap_err().0, + StatusCode::PAYLOAD_TOO_LARGE + ); + let mut invalid = payload(json!([])); + invalid.meta.app_name = "secret?token=value".into(); + assert_eq!( + sanitize_payload(invalid).unwrap_err().0, + StatusCode::BAD_REQUEST + ); +} + +#[test] +fn namespace_and_build_metadata_are_preserved_with_bounded_application_identity() { + let mut input = payload(json!([])); + input.meta.app_name = "canary".into(); + input.meta.app_version = "1.2.3+abc123".into(); + let output = sanitize_payload(input).unwrap(); + assert_eq!(output.meta.app_name, "rush-ui:canary"); + assert_eq!(output.meta.app_version, "1.2.3+abc123"); + assert_eq!(output.meta.environment, "production"); +} diff --git a/src/handlers/mod.rs b/src/handlers/mod.rs index af54a3f..2f3ef13 100644 --- a/src/handlers/mod.rs +++ b/src/handlers/mod.rs @@ -69,3 +69,5 @@ pub mod usage; pub mod usage_metering; pub mod users; pub mod whoami; + +pub mod frontend_telemetry; diff --git a/src/handlers/rum.rs b/src/handlers/rum.rs index f0ba2c9..a5bb6ef 100644 --- a/src/handlers/rum.rs +++ b/src/handlers/rum.rs @@ -260,7 +260,7 @@ pub struct RumSessionRow { /// serde_json::to_string solely to count bytes. /// Whether RUM is enabled (admin toggle in Settings → General). Defaults ON; /// only an explicit "false" disables it. Gates both RUM ingest endpoints. -async fn rum_enabled(state: &AppState) -> bool { +pub(super) async fn rum_enabled(state: &AppState) -> bool { state .config_db .get_setting("rum_enabled") @@ -283,6 +283,16 @@ pub async fn ingest( state.ingest_limits.check_decompressed("rum", body.len())?; let payload: RumIngestPayload = serde_json::from_slice(&body) .map_err(|_| state.ingest_limits.malformed("rum", "invalid RUM payload"))?; + ingest_events(state, tenant, payload, body.len()).await +} + +/// Shared durable RUM write path after the route-specific credential and payload checks. +pub(super) async fn ingest_events( + state: AppState, + tenant: TenantContext, + payload: RumIngestPayload, + wire_bytes: usize, +) -> Result { state .ingest_limits .check_entities("rum", payload.events.len())?; @@ -420,7 +430,7 @@ pub async fn ingest( tenant_id, "rum", payload.events.len() as u64, - body.len() as u64, + wire_bytes as u64, ); tracing::info!( diff --git a/src/main.rs b/src/main.rs index b1e60cd..d821b45 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1680,6 +1680,16 @@ fn consume_ingest_rate_limit(state: &AppState, key_id: &str, limit: u64) -> Resu Ok(()) } +fn should_reject_frontend_telemetry( + method: &Method, + path: &str, + credential: &CredentialKind, +) -> bool { + method == Method::POST + && path == "/api/v1/frontend/telemetry" + && *credential != CredentialKind::Session +} + fn credential_route_denial( credential: &CredentialKind, ingest_route: bool, @@ -1963,6 +1973,16 @@ async fn enforce_tenant_auth_middleware( _ => {} } + if should_reject_frontend_telemetry(&method, &path, &resolution.credential) { + // This narrow telemetry path deliberately rejects unauthenticated users + // even when a tenant allows anonymous queries. No generic ingest grant. + return ( + StatusCode::UNAUTHORIZED, + "frontend telemetry requires a browser session", + ) + .into_response(); + } + let agent_key = resolution .api_key .as_ref() @@ -3861,6 +3881,13 @@ async fn main() -> anyhow::Result<()> { get(handlers::groups::get_user_groups) .put(handlers::groups::set_user_groups), ) + // Dedicated session-authenticated, restricted health telemetry for this UI. + // General RUM/telemetry ingestion continues to require ingest credentials. + .route( + "/api/v1/frontend/telemetry", + post(handlers::frontend_telemetry::ingest) + .layer(DefaultBodyLimit::max(handlers::frontend_telemetry::MAX_BODY_BYTES)), + ) // RUM (Real User Monitoring) .route("/api/v1/rum/ingest", post(handlers::rum::ingest)) .route("/api/v1/rum/apps", get(handlers::rum::list_apps)) @@ -4470,6 +4497,42 @@ mod tenant_auth_tests { use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; + #[test] + fn frontend_health_is_a_separate_session_only_contract() { + let path = "/api/v1/frontend/telemetry"; + assert_eq!(ingest_signal_for_route(&Method::POST, path), None); + assert!(!super::should_reject_frontend_telemetry( + &Method::POST, + path, + &CredentialKind::Session + )); + for kind in [ + CredentialKind::Anonymous, + CredentialKind::QueryKey, + CredentialKind::IngestKey, + CredentialKind::InternalSre, + ] { + assert!(super::should_reject_frontend_telemetry( + &Method::POST, + path, + &kind + )); + } + assert!(requires_csrf_origin( + &Method::POST, + path, + Some(&CredentialKind::Session) + )); + assert_eq!( + credential_route_denial(&CredentialKind::Session, true, false), + Some("ingest_key_required") + ); + assert_eq!( + credential_route_denial(&CredentialKind::Session, true, true), + Some("ingest_key_required") + ); + } + #[test] fn startup_validates_api_key_secret_before_running_migrations() { let source = include_str!("main.rs"); diff --git a/tests/fixtures/frontend-clickhouse.xml b/tests/fixtures/frontend-clickhouse.xml new file mode 100644 index 0000000..afe43d5 --- /dev/null +++ b/tests/fixtures/frontend-clickhouse.xml @@ -0,0 +1,19 @@ + + rush_ + + + + local + /var/lib/clickhouse/frontend_test_cold/ + + + + + + default + frontend_test_cold + + + + + diff --git a/tests/fixtures/frontend-health-all-events.json b/tests/fixtures/frontend-health-all-events.json new file mode 100644 index 0000000..f5eb81c --- /dev/null +++ b/tests/fixtures/frontend-health-all-events.json @@ -0,0 +1,112 @@ +{ + "meta": { + "app_name": "health-smoke", + "app_version": "0.1.17", + "environment": "test", + "session_id": "fixture-session-0", + "user_id": "fixture-user", + "page_url": "https://frontend.example.test/logs/fixture?secret=fixture", + "page_path": "/logs/fixture?secret=fixture", + "view_name": "Fixture page", + "referrer": "https://frontend.example.test/fixture", + "browser_name": "Chrome", + "browser_version": "157.0.0.0", + "os_name": "macOS", + "os_version": "10.15.7", + "device_type": "desktop", + "screen_width": 1280, + "screen_height": 720 + }, + "events": [ + { + "event_type": "custom", + "event_name": "frontend_api_request", + "timestamp": 1794780000000000000, + "attributes": "{\"operation\":\"auth\",\"method\":\"GET\",\"outcome\":\"success\",\"status_class\":\"1xx\",\"cancel_reason\":\"none\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":42,\"duration_ms_max\":42,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}" + }, + { + "event_type": "custom", + "event_name": "frontend_api_attempt", + "timestamp": 1794780000000000000, + "attributes": "{\"operation\":\"auth\",\"method\":\"GET\",\"outcome\":\"success\",\"status_class\":\"1xx\",\"cancel_reason\":\"none\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":42,\"duration_ms_max\":42,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}" + }, + { + "event_type": "custom", + "event_name": "frontend_api_retry", + "timestamp": 1794780000000000000, + "attributes": "{\"operation\":\"auth\",\"method\":\"GET\",\"outcome\":\"success\",\"status_class\":\"1xx\",\"cancel_reason\":\"none\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":42,\"duration_ms_max\":42,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}" + }, + { + "event_type": "custom", + "event_name": "frontend_api_body", + "timestamp": 1794780000000000000, + "attributes": "{\"operation\":\"auth\",\"method\":\"GET\",\"outcome\":\"success\",\"status_class\":\"1xx\",\"cancel_reason\":\"none\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":42,\"duration_ms_max\":42,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}" + }, + { + "event_type": "custom", + "event_name": "frontend_started", + "timestamp": 1794780000000000000, + "attributes": "{\"startup_duration_ms\":42}" + }, + { + "event_type": "custom", + "event_name": "frontend_telemetry_dropped", + "timestamp": 1794780000000000000, + "attributes": "{\"count\":1}" + }, + { + "event_type": "custom", + "event_name": "frontend_exception", + "timestamp": 1794780000000000000, + "attributes": "{\"source\":\"vue\",\"stage\":\"setup\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":42,\"duration_ms_max\":42,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}" + }, + { + "event_type": "custom", + "event_name": "frontend_navigation", + "timestamp": 1794780000000000000, + "attributes": "{\"route\":\"alert-channel-add\",\"outcome\":\"success\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":42,\"duration_ms_max\":42,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}" + }, + { + "event_type": "custom", + "event_name": "frontend_investigation_stream", + "timestamp": 1794780000000000000, + "attributes": "{\"outcome\":\"completed\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":42,\"duration_ms_max\":42,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}" + }, + { + "event_type": "custom", + "event_name": "frontend_investigation_first_response", + "timestamp": 1794780000000000000, + "attributes": "{\"kind\":\"report\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":42,\"duration_ms_max\":42,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}" + }, + { + "event_type": "custom", + "event_name": "frontend_investigation_discarded", + "timestamp": 1794780000000000000, + "attributes": "{\"reason\":\"malformed\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":42,\"duration_ms_max\":42,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}" + }, + { + "event_type": "custom", + "event_name": "frontend_api_probe", + "timestamp": 1794780000000000000, + "attributes": "{\"outcome\":\"reachable\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":42,\"duration_ms_max\":42,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}" + }, + { + "event_type": "custom", + "event_name": "frontend_api_outage", + "timestamp": 1794780000000000000, + "attributes": "{\"reason\":\"offline\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":42,\"duration_ms_max\":42,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}" + }, + { + "event_type": "custom", + "event_name": "query_request_cache", + "timestamp": 1794780000000000000, + "attributes": "{\"cache_hits\":1,\"fresh_hits\":1,\"stale_hits\":1,\"revalidation_failures\":1,\"oversized_bypasses\":1,\"cache_entries\":1,\"cache_bytes\":1,\"in_flight\":1,\"in_flight_deduplications\":1,\"superseded_or_aborted\":1,\"cache_misses\":1,\"cache_evictions\":1}" + }, + { + "event_type": "custom", + "event_name": "polling_refresh", + "timestamp": 1794780000000000000, + "attributes": "{\"category\":\"alert_list\",\"refreshes\":1,\"failures\":1,\"skipped_overlaps\":1,\"backoffs\":1,\"average_duration_ms\":1,\"maximum_duration_ms\":1,\"maximum_backoff_ms\":1}" + } + ] +} diff --git a/tests/fixtures/frontend-health-sdk.json b/tests/fixtures/frontend-health-sdk.json new file mode 100644 index 0000000..b9b4109 --- /dev/null +++ b/tests/fixtures/frontend-health-sdk.json @@ -0,0 +1,297 @@ +[ + { + "meta": { + "app_name": "health-smoke", + "app_version": "0.1.17", + "environment": "test", + "session_id": "fixture-session-0", + "user_id": "fixture-user", + "page_url": "https://frontend.example.test/logs/fixture?secret=fixture", + "page_path": "/logs/fixture?secret=fixture", + "view_name": "Fixture page", + "referrer": "https://frontend.example.test/fixture", + "browser_name": "Chrome", + "browser_version": "157.0.0.0", + "os_name": "macOS", + "os_version": "10.15.7", + "device_type": "desktop", + "screen_width": 1280, + "screen_height": 720 + }, + "events": [ + { + "event_type": "pageview", + "event_name": "Fixture page", + "duration_ms": 58.39999997615814, + "trace_id": "fixture-trace", + "span_id": "fixture-span", + "timestamp": 1794780651734400000 + }, + { + "event_type": "custom", + "event_name": "frontend_started", + "attributes": "{\"startup_duration_ms\":84}", + "timestamp": 1794780651734500000 + }, + { + "event_type": "web_vital", + "vital_name": "FCP", + "vital_value": 80, + "vital_rating": "good", + "timestamp": 1794780651741599700 + }, + { + "event_type": "web_vital", + "vital_name": "TTFB", + "vital_value": 4.799999982118607, + "vital_rating": "good", + "timestamp": 1794780651741800000 + }, + { + "event_type": "pageview", + "event_name": "Fixture page", + "duration_ms": 58.39999997615814, + "trace_id": "fixture-trace", + "span_id": "fixture-span", + "timestamp": 1794780651742900000 + } + ] + }, + { + "meta": { + "app_name": "health-smoke", + "app_version": "0.1.17", + "environment": "test", + "session_id": "fixture-session-1", + "user_id": "fixture-user", + "page_url": "https://frontend.example.test/logs/fixture?secret=fixture", + "page_path": "/logs/fixture?secret=fixture", + "view_name": "Fixture page", + "referrer": "https://frontend.example.test/fixture", + "browser_name": "Chrome", + "browser_version": "157.0.0.0", + "os_name": "macOS", + "os_version": "10.15.7", + "device_type": "desktop", + "screen_width": 1280, + "screen_height": 720 + }, + "events": [ + { + "event_type": "interaction", + "interaction_type": "click", + "interaction_target": "a", + "timestamp": 1794780652536400000 + }, + { + "event_type": "web_vital", + "vital_name": "LCP", + "vital_value": 128, + "vital_rating": "good", + "timestamp": 1794780652572500000 + } + ] + }, + { + "meta": { + "app_name": "health-smoke", + "app_version": "0.1.17", + "environment": "test", + "session_id": "fixture-session-2", + "user_id": "fixture-user", + "page_url": "https://frontend.example.test/logs/fixture?secret=fixture", + "page_path": "/logs/fixture?secret=fixture", + "view_name": "Fixture page", + "referrer": "https://frontend.example.test/fixture", + "browser_name": "Chrome", + "browser_version": "157.0.0.0", + "os_name": "macOS", + "os_version": "10.15.7", + "device_type": "desktop", + "screen_width": 1280, + "screen_height": 720 + }, + "events": [ + { + "event_type": "pageview", + "event_name": "Fixture page", + "duration_ms": 58.39999997615814, + "trace_id": "fixture-trace", + "span_id": "fixture-span", + "timestamp": 1794780652964000000 + } + ] + }, + { + "meta": { + "app_name": "health-smoke", + "app_version": "0.1.17", + "environment": "test", + "session_id": "fixture-session-3", + "user_id": "fixture-user", + "page_url": "https://frontend.example.test/logs/fixture?secret=fixture", + "page_path": "/logs/fixture?secret=fixture", + "view_name": "Fixture page", + "referrer": "https://frontend.example.test/fixture", + "browser_name": "Chrome", + "browser_version": "157.0.0.0", + "os_name": "macOS", + "os_version": "10.15.7", + "device_type": "desktop", + "screen_width": 1280, + "screen_height": 720 + }, + "events": [ + { + "event_type": "custom", + "event_name": "frontend_api_attempt", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"features\",\"outcome\":\"success\",\"status_class\":\"2xx\",\"count\":2,\"duration_count\":2,\"duration_ms_sum\":50.099999994039536,\"duration_ms_max\":44.5,\"duration_ms_le_50\":2,\"duration_ms_le_100\":2,\"duration_ms_le_250\":2,\"duration_ms_le_500\":2,\"duration_ms_le_1000\":2,\"duration_ms_le_2500\":2,\"duration_ms_le_5000\":2,\"duration_ms_le_10000\":2,\"duration_ms_le_30000\":2,\"duration_ms_le_60000\":2}", + "timestamp": 1794780652964899800 + }, + { + "event_type": "custom", + "event_name": "frontend_api_attempt", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"license\",\"outcome\":\"success\",\"status_class\":\"2xx\",\"count\":2,\"duration_count\":2,\"duration_ms_sum\":52.90000003576279,\"duration_ms_max\":47.10000002384186,\"duration_ms_le_50\":2,\"duration_ms_le_100\":2,\"duration_ms_le_250\":2,\"duration_ms_le_500\":2,\"duration_ms_le_1000\":2,\"duration_ms_le_2500\":2,\"duration_ms_le_5000\":2,\"duration_ms_le_10000\":2,\"duration_ms_le_30000\":2,\"duration_ms_le_60000\":2}", + "timestamp": 1794780652965000000 + }, + { + "event_type": "custom", + "event_name": "frontend_api_attempt", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"auth\",\"outcome\":\"success\",\"status_class\":\"2xx\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":46.900000005960464,\"duration_ms_max\":46.900000005960464,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965099800 + }, + { + "event_type": "custom", + "event_name": "frontend_api_body", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"features\",\"outcome\":\"completed\",\"status_class\":\"2xx\",\"count\":2,\"duration_count\":2,\"duration_ms_sum\":6.299999982118607,\"duration_ms_max\":4.799999982118607,\"duration_ms_le_50\":2,\"duration_ms_le_100\":2,\"duration_ms_le_250\":2,\"duration_ms_le_500\":2,\"duration_ms_le_1000\":2,\"duration_ms_le_2500\":2,\"duration_ms_le_5000\":2,\"duration_ms_le_10000\":2,\"duration_ms_le_30000\":2,\"duration_ms_le_60000\":2}", + "timestamp": 1794780652965099800 + }, + { + "event_type": "custom", + "event_name": "frontend_api_body", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"license\",\"outcome\":\"completed\",\"status_class\":\"2xx\",\"count\":2,\"duration_count\":2,\"duration_ms_sum\":2.9999999701976776,\"duration_ms_max\":1.699999988079071,\"duration_ms_le_50\":2,\"duration_ms_le_100\":2,\"duration_ms_le_250\":2,\"duration_ms_le_500\":2,\"duration_ms_le_1000\":2,\"duration_ms_le_2500\":2,\"duration_ms_le_5000\":2,\"duration_ms_le_10000\":2,\"duration_ms_le_30000\":2,\"duration_ms_le_60000\":2}", + "timestamp": 1794780652965099800 + }, + { + "event_type": "custom", + "event_name": "frontend_api_body", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"auth\",\"outcome\":\"completed\",\"status_class\":\"2xx\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":1.199999988079071,\"duration_ms_max\":1.199999988079071,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965099800 + }, + { + "event_type": "custom", + "event_name": "frontend_api_request", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"auth\",\"outcome\":\"success\",\"status_class\":\"2xx\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":49.29999998211861,\"duration_ms_max\":49.29999998211861,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965099800 + }, + { + "event_type": "custom", + "event_name": "frontend_api_probe", + "attributes": "{\"outcome\":\"reachable\",\"count\":2,\"duration_count\":2,\"duration_ms_sum\":56.599999994039536,\"duration_ms_max\":54,\"duration_ms_le_100\":2,\"duration_ms_le_250\":2,\"duration_ms_le_500\":2,\"duration_ms_le_1000\":2,\"duration_ms_le_2500\":2,\"duration_ms_le_5000\":2,\"duration_ms_le_10000\":2,\"duration_ms_le_30000\":2,\"duration_ms_le_60000\":2,\"duration_ms_le_50\":1}", + "timestamp": 1794780652965099800 + }, + { + "event_type": "custom", + "event_name": "frontend_api_attempt", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"tenants\",\"outcome\":\"success\",\"status_class\":\"2xx\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":5,\"duration_ms_max\":5,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965099800 + }, + { + "event_type": "custom", + "event_name": "frontend_api_body", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"tenants\",\"outcome\":\"completed\",\"status_class\":\"2xx\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":0.30000001192092896,\"duration_ms_max\":0.30000001192092896,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965099800 + }, + { + "event_type": "custom", + "event_name": "frontend_navigation", + "attributes": "{\"outcome\":\"success\",\"route\":\"investigate\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":61.5,\"duration_ms_max\":61.5,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965099800 + }, + { + "event_type": "custom", + "event_name": "frontend_api_attempt", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"sessions\",\"outcome\":\"success\",\"status_class\":\"2xx\",\"count\":2,\"duration_count\":2,\"duration_ms_sum\":9.699999988079071,\"duration_ms_max\":6.699999988079071,\"duration_ms_le_50\":2,\"duration_ms_le_100\":2,\"duration_ms_le_250\":2,\"duration_ms_le_500\":2,\"duration_ms_le_1000\":2,\"duration_ms_le_2500\":2,\"duration_ms_le_5000\":2,\"duration_ms_le_10000\":2,\"duration_ms_le_30000\":2,\"duration_ms_le_60000\":2}", + "timestamp": 1794780652965200000 + }, + { + "event_type": "custom", + "event_name": "frontend_api_body", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"sessions\",\"outcome\":\"completed\",\"status_class\":\"2xx\",\"count\":2,\"duration_count\":2,\"duration_ms_sum\":0.20000001788139343,\"duration_ms_max\":0.10000002384185791,\"duration_ms_le_50\":2,\"duration_ms_le_100\":2,\"duration_ms_le_250\":2,\"duration_ms_le_500\":2,\"duration_ms_le_1000\":2,\"duration_ms_le_2500\":2,\"duration_ms_le_5000\":2,\"duration_ms_le_10000\":2,\"duration_ms_le_30000\":2,\"duration_ms_le_60000\":2}", + "timestamp": 1794780652965200000 + }, + { + "event_type": "custom", + "event_name": "frontend_api_request", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"GET\",\"operation\":\"sessions\",\"outcome\":\"success\",\"status_class\":\"2xx\",\"count\":2,\"duration_count\":2,\"duration_ms_sum\":9.900000005960464,\"duration_ms_max\":6.799999982118607,\"duration_ms_le_50\":2,\"duration_ms_le_100\":2,\"duration_ms_le_250\":2,\"duration_ms_le_500\":2,\"duration_ms_le_1000\":2,\"duration_ms_le_2500\":2,\"duration_ms_le_5000\":2,\"duration_ms_le_10000\":2,\"duration_ms_le_30000\":2,\"duration_ms_le_60000\":2}", + "timestamp": 1794780652965200000 + }, + { + "event_type": "custom", + "event_name": "frontend_api_attempt", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"POST\",\"operation\":\"investigate\",\"outcome\":\"success\",\"status_class\":\"2xx\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":2.9000000059604645,\"duration_ms_max\":2.9000000059604645,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965200000 + }, + { + "event_type": "custom", + "event_name": "frontend_api_body", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"POST\",\"operation\":\"investigate\",\"outcome\":\"completed\",\"status_class\":\"2xx\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":0.19999998807907104,\"duration_ms_max\":0.19999998807907104,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965200000 + }, + { + "event_type": "custom", + "event_name": "frontend_investigation_discarded", + "attributes": "{\"reason\":\"malformed\",\"count\":1}", + "timestamp": 1794780652965200000 + }, + { + "event_type": "custom", + "event_name": "frontend_investigation_first_response", + "attributes": "{\"kind\":\"evidence\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":3.199999988079071,\"duration_ms_max\":3.199999988079071,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965300000 + }, + { + "event_type": "custom", + "event_name": "frontend_investigation_stream", + "attributes": "{\"outcome\":\"completed\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":3.4000000059604645,\"duration_ms_max\":3.4000000059604645,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965300000 + }, + { + "event_type": "custom", + "event_name": "frontend_api_attempt", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"POST\",\"operation\":\"auth\",\"outcome\":\"success\",\"status_class\":\"2xx\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":35.79999998211861,\"duration_ms_max\":35.79999998211861,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965300000 + }, + { + "event_type": "custom", + "event_name": "frontend_api_body", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"POST\",\"operation\":\"auth\",\"outcome\":\"completed\",\"status_class\":\"2xx\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":0.7000000178813934,\"duration_ms_max\":0.7000000178813934,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965300000 + }, + { + "event_type": "custom", + "event_name": "frontend_api_request", + "attributes": "{\"cancel_reason\":\"none\",\"method\":\"POST\",\"operation\":\"auth\",\"outcome\":\"success\",\"status_class\":\"2xx\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":36.80000001192093,\"duration_ms_max\":36.80000001192093,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965300000 + }, + { + "event_type": "custom", + "event_name": "frontend_exception", + "attributes": "{\"source\":\"router\",\"stage\":\"lazy_chunk\",\"count\":1}", + "timestamp": 1794780652965300000 + }, + { + "event_type": "custom", + "event_name": "frontend_navigation", + "attributes": "{\"outcome\":\"failed\",\"route\":\"services\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":36.89999997615814,\"duration_ms_max\":36.89999997615814,\"duration_ms_le_50\":1,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965300000 + }, + { + "event_type": "custom", + "event_name": "frontend_api_outage", + "attributes": "{\"reason\":\"offline\",\"count\":1,\"duration_count\":1,\"duration_ms_sum\":61.400000005960464,\"duration_ms_max\":61.400000005960464,\"duration_ms_le_100\":1,\"duration_ms_le_250\":1,\"duration_ms_le_500\":1,\"duration_ms_le_1000\":1,\"duration_ms_le_2500\":1,\"duration_ms_le_5000\":1,\"duration_ms_le_10000\":1,\"duration_ms_le_30000\":1,\"duration_ms_le_60000\":1}", + "timestamp": 1794780652965300000 + } + ] + } +] diff --git a/tests/frontend_telemetry_session.rs b/tests/frontend_telemetry_session.rs new file mode 100644 index 0000000..bad5057 --- /dev/null +++ b/tests/frontend_telemetry_session.rs @@ -0,0 +1,335 @@ +//! Exercises the real binary, session cookies and production middleware against a +//! disposable ClickHouse supplied through RUSH_TEST_CLICKHOUSE_URL. Never use a +//! development/production database: the API bootstraps its test admin and schema. +use anyhow::{Context, Result}; +use reqwest::{Client, StatusCode, header}; +use serde_json::{Value, json}; +use std::{ + path::PathBuf, + process::{Child, Command, Stdio}, + time::{Duration, Instant}, +}; +struct ApiProcess { + child: Child, + root: PathBuf, +} +impl Drop for ApiProcess { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + let _ = std::fs::remove_dir_all(&self.root); + } +} +async fn login(client: &Client, base: &str, user: &str, password: &str) -> Result { + let response = client + .post(format!("{base}/api/v1/auth/login")) + .header(header::ORIGIN, base) + .json(&json!({"username":user,"password":password})) + .send() + .await?; + anyhow::ensure!( + response.status() == StatusCode::OK, + "test login failed: {}", + response.status() + ); + Ok(response + .headers() + .get(header::SET_COOKIE) + .context("session cookie")? + .to_str()? + .split(';') + .next() + .context("cookie value")? + .into()) +} +#[tokio::test] +#[ignore = "requires an empty disposable ClickHouse; starts the real API with isolated test-only secrets"] +async fn session_cookie_delivery_preserves_general_ingest_isolation() -> Result<()> { + let clickhouse = std::env::var("RUSH_TEST_CLICKHOUSE_URL")?; + let target = url::Url::parse(&clickhouse)?; + anyhow::ensure!( + target.scheme() == "http" + && target.host_str().is_some_and(|host| host == "localhost" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback())), + "test ClickHouse must be loopback HTTP" + ); + let probe = Client::builder().timeout(Duration::from_secs(5)).build()?; + let tables = probe.post(&clickhouse).body("SELECT count() FROM system.tables WHERE database NOT IN ('system', 'INFORMATION_SCHEMA', 'information_schema')").send().await?.error_for_status()?.text().await?; + anyhow::ensure!( + tables.trim() == "0", + "refusing to bootstrap a nonempty ClickHouse; run the disposable fixture script" + ); + + let root = std::env::temp_dir().join(format!("rush-frontend-auth-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&root)?; + let socket = std::net::TcpListener::bind("127.0.0.1:0")?; + let port = socket.local_addr()?.port(); + drop(socket); + let base = format!("http://127.0.0.1:{port}"); + let log = std::fs::File::create(root.join("api.log"))?; + let password = format!("Example-Widget-{}!", uuid::Uuid::new_v4()); + let mut command = Command::new(env!("CARGO_BIN_EXE_rush-api")); + command + .current_dir(&root) + .env_clear() + .stdout(Stdio::from(log.try_clone()?)) + .stderr(Stdio::from(log)); + for (key, value) in [ + ("RUSH_ENVIRONMENT", "test"), + ("RUSH_INSECURE_COOKIES", "true"), + ("RUSH_ALLOW_INSECURE_TENANT_READS", "true"), + ("RUSH_SSO_REPLAY_STORE", "local"), + ("RUSH_INGEST_BATCH_ROWS", "1"), + ("RUSH_CLICKHOUSE_ASYNC_INSERT", "false"), + ("RUSH_RUN_REPLAYER", "false"), + ("RUSH_RUN_SLO_ENGINE", "false"), + ("RUSH_RUN_ANOMALY_ENGINE", "false"), + ("RUSH_RUN_MONITOR_ENGINE", "false"), + ("RUSH_RUN_SIEM_ENGINE", "false"), + ("RUSH_RUN_TELEMETRY_QUALITY_ENGINE", "false"), + ("INITIAL_ADMIN_PASSWORD", password.as_str()), + ("RUST_LOG", "warn"), + ] { + command.env(key, value); + } + for key in [ + "RUSH_API_KEY_SECRET", + "RUSH_SESSION_HMAC_SECRET", + "RUSH_AUDIT_HMAC_SECRET", + "RUSH_CONFIG_ENCRYPTION_KEY", + "RUSH_SSO_TRANSACTION_SECRET", + ] { + command.env(key, uuid::Uuid::new_v4().to_string()); + } + command + .env("CLICKHOUSE_URL", &clickhouse) + .env("RUSH_PORT", port.to_string()) + .env("RUSH_BASE_URL", &base) + .env("TMPDIR", &root); + let mut api = ApiProcess { + child: command.spawn()?, + root, + }; + let client = Client::builder().timeout(Duration::from_secs(10)).build()?; + let deadline = Instant::now() + Duration::from_secs(120); + loop { + if let Some(status) = api.child.try_wait()? { + anyhow::bail!( + "isolated API exited {status}: {}", + std::fs::read_to_string(api.root.join("api.log"))? + ); + } + if client + .get(format!("{base}/healthz")) + .send() + .await + .is_ok_and(|r| r.status().is_success()) + { + break; + } + anyhow::ensure!(Instant::now() < deadline, "isolated API startup timeout"); + tokio::time::sleep(Duration::from_millis(200)).await; + } + let cookie = login(&client, &base, "admin", &password).await?; + let payload = json!({"meta":{"app_name":"rush-ui","app_version":"1.2.3","environment":"test","page_path":"/logs/private-id?token=private-query","page_url":"https://private-host/logs?token=private-query","session_id":"private-session","user_id":"private-user"},"events":[{"event_type":"custom","event_name":"frontend_api_request","attributes":json!({"operation":"query","method":"POST","outcome":"success","count":1,"duration_ms_sum":42,"query":"private-query"}).to_string()}]}); + let request = |path: &str| { + client + .post(format!("{base}{path}")) + .header(header::ORIGIN, &base) + .header(header::COOKIE, &cookie) + }; + for content_type in ["application/json", "text/plain;charset=UTF-8"] { + let response = request("/api/v1/frontend/telemetry") + .header(header::CONTENT_TYPE, content_type) + .body(payload.to_string()) + .send() + .await?; + anyhow::ensure!( + response.status() == StatusCode::OK, + "frontend health request failed: {}", + response.status() + ); + assert_eq!(response.json::().await?["accepted"], 1); + } + assert_eq!( + request("/api/v1/rum/ingest") + .json(&payload) + .send() + .await? + .status(), + StatusCode::FORBIDDEN + ); + for forged in ["", "rush_session=forged-session"] { + assert_eq!( + client + .post(format!("{base}/api/v1/frontend/telemetry")) + .header(header::ORIGIN, &base) + .header(header::COOKIE, forged) + .header("x-rush-credential-type", "session") + .json(&payload) + .send() + .await? + .status(), + StatusCode::UNAUTHORIZED + ); + } + for content_type in ["application/json", "text/plain;charset=UTF-8"] { + for origin in [Some("https://attacker.invalid"), None] { + let mut forged = client + .post(format!("{base}/api/v1/frontend/telemetry")) + .header(header::COOKIE, &cookie) + .header(header::CONTENT_TYPE, content_type) + .body(payload.to_string()); + if let Some(origin) = origin { + forged = forged.header(header::ORIGIN, origin); + } + assert_eq!(forged.send().await?.status(), StatusCode::FORBIDDEN); + } + } + let other = request("/api/v1/tenants") + .json(&json!({"name":"frontend-other"})) + .send() + .await?; + anyhow::ensure!(other.status().is_success(), "create isolated tenant failed"); + assert_eq!( + request("/api/v1/frontend/telemetry") + .header("x-rush-tenant", "frontend-other") + .json(&payload) + .send() + .await? + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + request("/api/v1/frontend/telemetry") + .header(header::CONTENT_TYPE, "application/json") + .body("x".repeat(64 * 1024 + 1)) + .send() + .await? + .status(), + StatusCode::PAYLOAD_TOO_LARGE + ); + anyhow::ensure!( + client + .put(format!("{base}/api/v1/groups/viewers/tenants")) + .header(header::ORIGIN, &base) + .header(header::COOKIE, &cookie) + .json(&json!({"tenant_ids":["default"]})) + .send() + .await? + .status() + .is_success(), + "grant isolated viewer default-tenant access failed" + ); + let viewer_password = format!("Example-Viewer-{}!", uuid::Uuid::new_v4()); + let viewer = request("/api/v1/users") + .json(&json!({"username":"frontendviewer", "password":viewer_password})) + .send() + .await?; + anyhow::ensure!( + viewer.status().is_success(), + "create isolated viewer failed: {}", + viewer.status() + ); + let viewer_cookie = login(&client, &base, "frontendviewer", &viewer_password).await?; + let viewer_request = || { + client + .post(format!("{base}/api/v1/frontend/telemetry")) + .header(header::ORIGIN, &base) + .header(header::COOKIE, &viewer_cookie) + .json(&payload) + }; + assert_eq!(viewer_request().send().await?.status(), StatusCode::OK); + assert_eq!( + viewer_request() + .header("x-rush-tenant", "frontend-other") + .send() + .await? + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + client + .post(format!("{base}/api/v1/rum/ingest")) + .header(header::ORIGIN, &base) + .header(header::COOKIE, &viewer_cookie) + .json(&payload) + .send() + .await? + .status(), + StatusCode::FORBIDDEN + ); + let rum_setting = |enabled: bool| { + client + .put(format!("{base}/api/v1/settings/rum")) + .header(header::ORIGIN, &base) + .header(header::COOKIE, &cookie) + .json(&json!({"enabled":enabled})) + }; + anyhow::ensure!( + rum_setting(false).send().await?.status().is_success(), + "disable isolated RUM failed" + ); + assert_eq!( + viewer_request().send().await?.status(), + StatusCode::FORBIDDEN + ); + anyhow::ensure!( + rum_setting(true).send().await?.status().is_success(), + "restore isolated RUM failed" + ); + let rum_signal = |enabled: bool| { + client + .put(format!("{base}/api/v1/tenants/default/signals")) + .header(header::ORIGIN, &base) + .header(header::COOKIE, &cookie) + .json(&json!({"rum":enabled})) + }; + anyhow::ensure!( + rum_signal(false).send().await?.status().is_success(), + "disable isolated tenant RUM signal failed" + ); + assert_eq!(viewer_request().send().await?.status(), StatusCode::OK); + anyhow::ensure!( + rum_signal(true).send().await?.status().is_success(), + "restore isolated tenant RUM signal failed" + ); + // Synchronous insertion lets this confirm actual sanitized delivery, not just HTTP acceptance. + let rows=client.post(&clickhouse).body("SELECT tenant_id, AppName, AppVersion, Environment, PagePath, PageUrl, SessionId, UserId, EventName, Attributes FROM observability.rum FORMAT JSONEachRow").send().await?.error_for_status()?.text().await?; + assert!(!rows.contains("private-"), "raw private data reached RUM"); + let rows = rows + .lines() + .map(serde_json::from_str::) + .collect::, _>>()?; + assert_eq!(rows.len(), 3); + for row in rows { + assert_eq!(row["tenant_id"], "default"); + assert_eq!(row["AppName"], "rush-ui"); + assert_eq!(row["AppVersion"], "1.2.3"); + assert_eq!(row["Environment"], "test"); + assert_eq!(row["PagePath"], "/logs"); + let attrs: Value = serde_json::from_str(row["Attributes"].as_str().unwrap())?; + assert_eq!(attrs["count"], 1); + assert_eq!(attrs["duration_ms_sum"], 42); + } + // Logging out revokes the exact cookie used by the next telemetry request. + anyhow::ensure!( + request("/api/v1/auth/logout") + .send() + .await? + .status() + .is_success(), + "logout failed" + ); + assert_eq!( + request("/api/v1/frontend/telemetry") + .json(&payload) + .send() + .await? + .status(), + StatusCode::UNAUTHORIZED + ); + Ok(()) +} diff --git a/tests/run-frontend-telemetry.sh b/tests/run-frontend-telemetry.sh new file mode 100644 index 0000000..243ea8e --- /dev/null +++ b/tests/run-frontend-telemetry.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" +container="rush-frontend-telemetry-test-$$" +cleanup() { docker rm -f "$container" >/dev/null 2>&1 || true; } +trap cleanup EXIT +# Pinned official ClickHouse 26.10.3.7; all API credentials are isolated test +# values generated inside the integration test, never read from the developer shell. +docker run -d --rm --name "$container" --memory=1536m --cpus=2 \ + -p 127.0.0.1::8123 -e CLICKHOUSE_SKIP_USER_SETUP=1 \ + --mount "type=bind,source=$repo_root/tests/fixtures/frontend-clickhouse.xml,target=/etc/clickhouse-server/config.d/frontend-test.xml,readonly" \ + clickhouse/clickhouse-server@sha256:739e099abd1157390960a59e4532d21781d8fde57e6e4151cb920ea02c30b1ce >/dev/null +address="$(docker port "$container" 8123/tcp)" +export RUSH_TEST_CLICKHOUSE_URL="http://$address" +ready=false +for attempt in {1..60}; do + if curl --fail --silent --max-time 2 "$RUSH_TEST_CLICKHOUSE_URL/ping" >/dev/null; then ready=true; break; fi + sleep 1 +done +if [[ "$ready" != true ]]; then docker logs "$container"; exit 1; fi +cargo test --test frontend_telemetry_session --all-features -- --ignored