diff --git a/src/anomaly_engine.rs b/src/anomaly_engine.rs index 8f4b113..c5c4466 100644 --- a/src/anomaly_engine.rs +++ b/src/anomaly_engine.rs @@ -1,5 +1,6 @@ use crate::alert_engine::SmtpConfig; use crate::clickhouse_config::ConfigDb; +use crate::engine_metrics::{EvaluationObservation, EvaluationOutcome}; use crate::models::anomaly::AnomalyRule; use crate::query_builder::ERROR_SPAN; use chrono::DurationRound; @@ -192,6 +193,7 @@ async fn eval_anomaly_rules( let now = chrono::Utc::now(); let now_str = now.format("%Y-%m-%dT%H:%M:%SZ").to_string(); let due_rules = config_db.get_due_anomaly_rules(&now_str).await?; + crate::engine_metrics::due(config_db.self_metrics(), "anomaly_engine", due_rules.len()); if due_rules.is_empty() { tracing::debug!(engine = "anomaly", "tick -- no rules due"); @@ -206,6 +208,8 @@ async fn eval_anomaly_rules( // Prefetch every due rule's data concurrently — the fetches are independent // reads, so the tick pays for the slowest fetch instead of the sum of all. let fetched = futures_util::future::join_all(due_rules.iter().map(|rule| async move { + let mut observation = + EvaluationObservation::new(config_db.self_metrics(), "anomaly_engine"); let fetched = crate::query_governor::run_background(&rule.tenant_id, async { match rule.source.as_str() { "apm" => Some(fetch_apm_data(read_ch, rule, &now).await), @@ -214,12 +218,17 @@ async fn eval_anomaly_rules( } }) .await; - match fetched { + let rejected = fetched.is_err(); + let result = match fetched { Ok(result) => result, - Err(error) => Some(Err(anyhow::anyhow!( - "background admission rejected: {error:?}" - ))), - } + Err(error) => { + observation.finish(EvaluationOutcome::AdmissionRejected); + Some(Err(anyhow::anyhow!( + "background admission rejected: {error:?}" + ))) + } + }; + (result, observation, rejected) })) .await; @@ -227,7 +236,11 @@ async fn eval_anomaly_rules( // at the end of the tick (previously one INSERT per series per rule). let mut eval_log_values: Vec = Vec::new(); - for (rule, prefetched) in due_rules.into_iter().zip(fetched) { + let mut failures = 0; + for (rule, (prefetched, mut observation, rejected)) in due_rules.into_iter().zip(fetched) { + if !rejected { + observation.finish(EvaluationOutcome::Error); + } tracing::debug!( engine = "anomaly", rule_name = %rule.name, @@ -239,6 +252,8 @@ async fn eval_anomaly_rules( let all_series = match prefetched { Some(r) => r, None => { + observation.finish(EvaluationOutcome::InvalidConfig); + failures += 1; tracing::warn!(engine = "anomaly", rule_id = %rule.id, source = %rule.source, "unknown data source"); continue; } @@ -251,9 +266,11 @@ async fn eval_anomaly_rules( config_db .update_anomaly_state(&rule.id, &rule.tenant_id, "no_data", &now_str, None) .await?; + observation.finish(EvaluationOutcome::NoData); continue; } Err(e) => { + failures += 1; tracing::warn!(error = %e, engine = "anomaly", rule_id = %rule.id, "data fetch failed"); config_db .update_anomaly_state(&rule.id, &rule.tenant_id, "no_data", &now_str, None) @@ -391,6 +408,7 @@ async fn eval_anomaly_rules( .update_anomaly_state(&rule.id, &rule.tenant_id, new_state, &now_str, None) .await?; } + observation.finish(EvaluationOutcome::Completed); } // Flush all per-series eval logs in one INSERT (previously one per series). @@ -400,10 +418,22 @@ async fn eval_anomaly_rules( eval_log_values.join(", ") ); if let Err(e) = write_ch.query(&sql).execute().await { + failures += 1; + if let Some(metrics) = config_db.self_metrics() { + metrics.inc_counter( + "rush_engine_persistence_failures_total", + &[("engine", "anomaly_engine"), ("operation", "eval_log")], + 1, + ); + } tracing::warn!(error = %e, engine = "anomaly", rows = eval_log_values.len(), "failed to write eval log batch"); } } + anyhow::ensure!( + failures == 0, + "{failures} anomaly evaluation operation(s) failed" + ); Ok(()) } diff --git a/src/bin/anomaly_engine.rs b/src/bin/anomaly_engine.rs index 35adce4..7e589c5 100644 --- a/src/bin/anomaly_engine.rs +++ b/src/bin/anomaly_engine.rs @@ -75,9 +75,10 @@ async fn main() -> anyhow::Result<()> { std::env::var("RUSH_PROM_BASE_URL").unwrap_or_else(|_| "http://localhost:8080".to_string()); tracing::info!("wide-anomaly-engine starting"); - // The standalone engine has no /metrics endpoint; give it a private registry so the - // engine-loop instrumentation still works (record_engine just updates in-memory atomics). + // The standalone worker exports the same bounded metrics as the API process. let self_metrics = Arc::new(rush_api::self_metrics::SelfMetrics::new()); + config_db.attach_metrics(self_metrics.clone()); + rush_api::process_metrics::spawn(self_metrics.clone()); let query_limits = rush_api::query_governor::QueryGovernorConfig::at_startup( config_db .get_setting(rush_api::query_governor::QUERY_LIMITS_SETTING_KEY) @@ -94,6 +95,8 @@ async fn main() -> anyhow::Result<()> { // Takes the same lease as the in-process engine, so this deployment and any // API replica can never both evaluate rules. let shutdown = rush_api::shutdown::ShutdownController::new(); + let _metrics_server = + rush_api::worker_metrics::spawn(self_metrics.clone(), shutdown.clone()).await?; let election = rush_api::leader::LeaderElection::from_env( config_db.client.clone(), &rush_api::stats_engine::configured_instance_id(), @@ -120,6 +123,7 @@ async fn main() -> anyhow::Result<()> { } } + shutdown.finish(); Ok(()) } diff --git a/src/clickhouse_config.rs b/src/clickhouse_config.rs index 725dca5..4f8f815 100644 --- a/src/clickhouse_config.rs +++ b/src/clickhouse_config.rs @@ -87,6 +87,15 @@ struct SavedQueryRow { } impl ConfigDb { + /// Attach the shared process registry before starting background work. + pub fn attach_metrics(&self, metrics: std::sync::Arc) { + let _ = self.metrics.set(metrics); + } + + pub fn self_metrics(&self) -> Option<&crate::self_metrics::SelfMetrics> { + self.metrics.get().map(std::sync::Arc::as_ref) + } + pub async fn create_investigation_session( &self, id: &str, @@ -2443,6 +2452,7 @@ mod sql_shape_tests { } pub struct ConfigDb { + metrics: std::sync::OnceLock>, pub client: Client, /// Tenant name-or-id → flags. Negative results are cached too so unknown /// tenants on the ingest path don't hammer ClickHouse. @@ -2635,6 +2645,7 @@ impl ConfigDb { .with_user(user) .with_password(password); let db = Self { + metrics: std::sync::OnceLock::new(), client, tenant_cache: DashMap::new(), ingest_auth_cache: DashMap::new(), @@ -12407,6 +12418,7 @@ mod session_tenant_regressions { fn config(mock: &Mock) -> ConfigDb { ConfigDb { + metrics: std::sync::OnceLock::new(), client: Client::default() .with_url(mock.url()) .with_compression(clickhouse::Compression::None), diff --git a/src/engine_metrics.rs b/src/engine_metrics.rs new file mode 100644 index 0000000..0d560fe --- /dev/null +++ b/src/engine_metrics.rs @@ -0,0 +1,130 @@ +//! Bounded evaluation outcomes, including cancelled work and successful no-data reads. +use crate::self_metrics::SelfMetrics; +use std::time::Instant; + +#[derive(Clone, Copy)] +pub enum EvaluationOutcome { + Completed, + NoData, + Error, + AdmissionRejected, + InvalidConfig, +} + +pub struct EvaluationObservation<'a> { + metrics: Option<&'a SelfMetrics>, + engine: &'static str, + started: Instant, + outcome: &'static str, +} + +impl<'a> EvaluationObservation<'a> { + pub fn new(metrics: Option<&'a SelfMetrics>, engine: &'static str) -> Self { + let engine = match engine { + "monitor_engine" | "siem_engine" | "slo_engine" | "anomaly_engine" => engine, + _ => "other", + }; + if let Some(metrics) = metrics { + metrics.add_gauge( + "rush_engine_evaluations_inflight", + &[("engine", engine)], + 1.0, + ); + } + Self { + metrics, + engine, + started: Instant::now(), + outcome: "cancelled", + } + } + pub fn finish(&mut self, outcome: EvaluationOutcome) { + self.outcome = match outcome { + EvaluationOutcome::Completed => "ok", + EvaluationOutcome::NoData => "no_data", + EvaluationOutcome::Error => "error", + EvaluationOutcome::AdmissionRejected => "admission_rejected", + EvaluationOutcome::InvalidConfig => "invalid_config", + }; + } +} + +impl Drop for EvaluationObservation<'_> { + fn drop(&mut self) { + if let Some(metrics) = self.metrics { + let labels = [("engine", self.engine), ("outcome", self.outcome)]; + metrics.add_gauge( + "rush_engine_evaluations_inflight", + &[("engine", self.engine)], + -1.0, + ); + metrics.inc_counter("rush_engine_evaluations_total", &labels, 1); + metrics.observe_histogram( + "rush_engine_evaluation_duration_ms", + &labels, + self.started.elapsed().as_secs_f64() * 1000.0, + ); + if matches!(self.outcome, "ok" | "no_data") { + metrics.set_gauge( + "rush_engine_evaluation_last_success_timestamp_seconds", + &[("engine", self.engine)], + chrono::Utc::now().timestamp() as f64, + ); + } + } + } +} + +pub fn due(metrics: Option<&SelfMetrics>, engine: &'static str, count: usize) { + if let Some(metrics) = metrics { + metrics.set_gauge( + "rush_engine_evaluations_due", + &[("engine", engine)], + count as f64, + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + #[tokio::test] + async fn cancellation_releases_inflight_and_is_not_a_success() { + let metrics = Arc::new(SelfMetrics::new()); + let registry = metrics.clone(); + let (tx, rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let _observation = EvaluationObservation::new(Some(®istry), "monitor_engine"); + tx.send(()).unwrap(); + std::future::pending::<()>().await; + }); + rx.await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + let text = metrics.render_prometheus(); + assert!(text.contains("rush_engine_evaluations_inflight{engine=\"monitor_engine\"} 0")); + assert!(text.contains( + "rush_engine_evaluations_total{engine=\"monitor_engine\",outcome=\"cancelled\"} 1" + )); + assert!(!text.contains("last_success")); + } + #[test] + fn failed_work_does_not_advance_last_success_and_labels_are_bounded() { + let metrics = SelfMetrics::new(); + { + let mut work = EvaluationObservation::new(Some(&metrics), "slo_engine"); + work.finish(EvaluationOutcome::NoData); + } + { + let mut work = EvaluationObservation::new(Some(&metrics), "raw-rule-name"); + work.finish(EvaluationOutcome::Error); + } + let text = metrics.render_prometheus(); + assert!(text.contains( + "rush_engine_evaluation_last_success_timestamp_seconds{engine=\"slo_engine\"}" + )); + assert!(!text.contains("last_success_timestamp_seconds{engine=\"other\"}")); + assert!(!text.contains("raw-rule-name")); + } +} diff --git a/src/lib.rs b/src/lib.rs index 265a874..9d195b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ pub mod config; pub mod cors; pub mod detection_query; pub mod edition; +pub mod engine_metrics; pub mod eval_state; pub mod github_repository_policy; pub mod handlers; diff --git a/src/main.rs b/src/main.rs index e01666d..561c5ff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2584,6 +2584,7 @@ async fn main() -> anyhow::Result<()> { // outbox degradation is visible even if no one has scraped `/metrics` yet. let self_metrics: std::sync::Arc = std::sync::Arc::new(rush_api::self_metrics::SelfMetrics::new()); + config_db.attach_metrics(self_metrics.clone()); let query_governor_config = QueryGovernorConfig::at_startup( config_db .get_setting(QUERY_LIMITS_SETTING_KEY) diff --git a/src/monitor_engine.rs b/src/monitor_engine.rs index 9d70e56..ae7acc7 100644 --- a/src/monitor_engine.rs +++ b/src/monitor_engine.rs @@ -1,3 +1,4 @@ +use crate::engine_metrics::{EvaluationObservation, EvaluationOutcome}; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; use std::time::Instant; @@ -342,8 +343,11 @@ async fn run_evaluation_cycle( let now_str_ref = now_str.as_str(); let monitor_states_ref = &monitor_states; - let outcomes: Vec<(String, u64, bool)> = + crate::engine_metrics::due(config_db.self_metrics(), "monitor_engine", jobs.len()); + let outcomes: Vec<(String, u64, bool, bool)> = futures_util::stream::iter(jobs.into_iter().map(|(monitor, should_flush)| async move { + let mut observation = + EvaluationObservation::new(config_db.self_metrics(), "monitor_engine"); let result = match crate::query_governor::run_background( &monitor.tenant_id, evaluate_monitor( @@ -361,9 +365,20 @@ async fn run_evaluation_cycle( ) .await { - Ok(result) => result, - Err(error) => Err(anyhow::anyhow!("background admission rejected: {error:?}")), + Ok(result) => { + observation.finish(if result.is_ok() { + EvaluationOutcome::Completed + } else { + EvaluationOutcome::Error + }); + result + } + Err(error) => { + observation.finish(EvaluationOutcome::AdmissionRejected); + Err(anyhow::anyhow!("background admission rejected: {error:?}")) + } }; + let failed = result.is_err(); let (changes, persisted) = match result { Ok(cp) => cp, Err(e) => { @@ -388,18 +403,21 @@ async fn run_evaluation_cycle( .await } }; - (monitor.id, changes, persisted) + (monitor.id, changes, persisted, failed) })) .buffer_unordered(ENGINE_CONCURRENCY) .collect() .await; let mut state_changes: u64 = 0; - for (id, changes, persisted) in outcomes { + let mut failures = 0; + for (id, changes, persisted, failed) in outcomes { + failures += usize::from(failed); state_changes += changes; eval_state.record(id, now, persisted); } + anyhow::ensure!(failures == 0, "{failures} monitor evaluation job(s) failed"); Ok((evaluated, state_changes)) } diff --git a/src/self_metrics.rs b/src/self_metrics.rs index 16e6718..b6a0e23 100644 --- a/src/self_metrics.rs +++ b/src/self_metrics.rs @@ -409,6 +409,13 @@ impl SelfMetrics { .map(|d| d.as_secs()) .unwrap_or(0); self.set_gauge("rush_engine_last_run_timestamp", &labels, now as f64); + if ok { + self.set_gauge( + "rush_engine_last_success_timestamp_seconds", + &labels, + now as f64, + ); + } } /// Convenience: record one log/span SEARCH and update all search self-metrics. diff --git a/src/siem_engine.rs b/src/siem_engine.rs index e4f27d1..613afb6 100644 --- a/src/siem_engine.rs +++ b/src/siem_engine.rs @@ -1,4 +1,5 @@ use crate::clickhouse_config::ConfigDb; +use crate::engine_metrics::{EvaluationObservation, EvaluationOutcome}; use crate::models::detection::DetectionRule; use clickhouse::Client; use std::sync::Arc; @@ -59,6 +60,7 @@ async fn run_detection_cycle( let rules = config_db.list_enabled_detection_rules().await?; if rules.is_empty() { + crate::engine_metrics::due(config_db.self_metrics(), "siem_engine", 0); tracing::debug!(engine = "siem", "tick -- no enabled detection rules"); return Ok(()); } @@ -76,11 +78,14 @@ async fn run_detection_cycle( }) .collect(); + crate::engine_metrics::due(config_db.self_metrics(), "siem_engine", jobs.len()); let evaluated = jobs.len() as u32; let now_str_ref = now_str.as_str(); - let outcomes: Vec<(String, bool, bool)> = + let outcomes: Vec<(String, bool, bool, bool)> = futures_util::stream::iter(jobs.into_iter().map(|(rule, should_flush)| async move { + let mut observation = + EvaluationObservation::new(config_db.self_metrics(), "siem_engine"); let evaluated = crate::query_governor::run_background( &rule.tenant_id, evaluate_rule( @@ -95,8 +100,12 @@ async fn run_detection_cycle( ) .await; match evaluated { - Ok(Ok((did_fire, persisted))) => (rule.id, did_fire, persisted), + Ok(Ok((did_fire, persisted))) => { + observation.finish(EvaluationOutcome::Completed); + (rule.id, did_fire, persisted, false) + } Ok(Err(e)) => { + observation.finish(EvaluationOutcome::Error); tracing::warn!( error = %e, engine = "siem", @@ -104,9 +113,10 @@ async fn run_detection_cycle( rule_id = %rule.id, "rule evaluation failed" ); - (rule.id, false, false) + (rule.id, false, false, true) } Err(e) => { + observation.finish(EvaluationOutcome::AdmissionRejected); tracing::warn!( error = ?e, engine = "siem", @@ -114,7 +124,7 @@ async fn run_detection_cycle( rule_id = %rule.id, "rule evaluation admission rejected" ); - (rule.id, false, false) + (rule.id, false, false, true) } } })) @@ -123,7 +133,9 @@ async fn run_detection_cycle( .await; let mut fired = 0u32; - for (id, did_fire, persisted) in outcomes { + let mut failures = 0; + for (id, did_fire, persisted, failed) in outcomes { + failures += usize::from(failed); if did_fire { fired += 1; } @@ -141,6 +153,10 @@ async fn run_detection_cycle( tracing::debug!(engine = "siem", "tick -- no rules due"); } + anyhow::ensure!( + failures == 0, + "{failures} detection evaluation job(s) failed" + ); Ok(()) } diff --git a/src/slo_engine.rs b/src/slo_engine.rs index 1c1de9b..404a04c 100644 --- a/src/slo_engine.rs +++ b/src/slo_engine.rs @@ -1,4 +1,5 @@ use crate::clickhouse_config::ConfigDb; +use crate::engine_metrics::{EvaluationObservation, EvaluationOutcome}; use crate::models::query::{Filter, FilterOp}; use crate::query_builder::{QueryClauses, build_metrics_where_clause, build_where_clause}; use clickhouse::Client; @@ -405,9 +406,12 @@ async fn eval_slos( }) .collect(); + crate::engine_metrics::due(config_db.self_metrics(), "slo_engine", jobs.len()); let now_str_ref = now_str.as_str(); let outcomes: Vec<(String, bool, bool)> = futures_util::stream::iter(jobs.into_iter().map(|(slo, should_flush)| async move { + let mut observation = + EvaluationObservation::new(config_db.self_metrics(), "slo_engine"); let evaluated = crate::query_governor::run_background( &slo.tenant_id, eval_one_slo( @@ -423,12 +427,17 @@ async fn eval_slos( ) .await; let (persisted, succeeded) = match evaluated { - Ok(Ok(p)) => (p, true), + Ok(Ok(p)) => { + observation.finish(EvaluationOutcome::Completed); + (p, true) + } Ok(Err(e)) => { + observation.finish(EvaluationOutcome::Error); tracing::warn!("slo {}: evaluation error: {e}", slo.id); (false, false) } Err(e) => { + observation.finish(EvaluationOutcome::AdmissionRejected); tracing::warn!("slo {}: background admission rejected: {e:?}", slo.id); (false, false) }