Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 36 additions & 6 deletions src/anomaly_engine.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::alert_engine::SmtpConfig;
use crate::clickhouse_config::ConfigDb;
use crate::engine_metrics::{EvaluationObservation, EvaluationOutcome};
use crate::models::anomaly::AnomalyRule;
use crate::query_builder::ERROR_SPAN;
use chrono::DurationRound;
Expand Down Expand Up @@ -192,6 +193,7 @@ async fn eval_anomaly_rules(
let now = chrono::Utc::now();
let now_str = now.format("%Y-%m-%dT%H:%M:%SZ").to_string();
let due_rules = config_db.get_due_anomaly_rules(&now_str).await?;
crate::engine_metrics::due(config_db.self_metrics(), "anomaly_engine", due_rules.len());

if due_rules.is_empty() {
tracing::debug!(engine = "anomaly", "tick -- no rules due");
Expand All @@ -206,6 +208,8 @@ async fn eval_anomaly_rules(
// Prefetch every due rule's data concurrently — the fetches are independent
// reads, so the tick pays for the slowest fetch instead of the sum of all.
let fetched = futures_util::future::join_all(due_rules.iter().map(|rule| async move {
let mut observation =
EvaluationObservation::new(config_db.self_metrics(), "anomaly_engine");
let fetched = crate::query_governor::run_background(&rule.tenant_id, async {
match rule.source.as_str() {
"apm" => Some(fetch_apm_data(read_ch, rule, &now).await),
Expand All @@ -214,20 +218,29 @@ async fn eval_anomaly_rules(
}
})
.await;
match fetched {
let rejected = fetched.is_err();
let result = match fetched {
Ok(result) => result,
Err(error) => Some(Err(anyhow::anyhow!(
"background admission rejected: {error:?}"
))),
}
Err(error) => {
observation.finish(EvaluationOutcome::AdmissionRejected);
Some(Err(anyhow::anyhow!(
"background admission rejected: {error:?}"
)))
}
};
(result, observation, rejected)
}))
.await;

// Per-series eval logs are accumulated here and written as ONE batch insert
// at the end of the tick (previously one INSERT per series per rule).
let mut eval_log_values: Vec<String> = Vec::new();

for (rule, prefetched) in due_rules.into_iter().zip(fetched) {
let mut failures = 0;
for (rule, (prefetched, mut observation, rejected)) in due_rules.into_iter().zip(fetched) {
if !rejected {
observation.finish(EvaluationOutcome::Error);
}
tracing::debug!(
engine = "anomaly",
rule_name = %rule.name,
Expand All @@ -239,6 +252,8 @@ async fn eval_anomaly_rules(
let all_series = match prefetched {
Some(r) => r,
None => {
observation.finish(EvaluationOutcome::InvalidConfig);
failures += 1;
tracing::warn!(engine = "anomaly", rule_id = %rule.id, source = %rule.source, "unknown data source");
continue;
}
Expand All @@ -251,9 +266,11 @@ async fn eval_anomaly_rules(
config_db
.update_anomaly_state(&rule.id, &rule.tenant_id, "no_data", &now_str, None)
.await?;
observation.finish(EvaluationOutcome::NoData);
continue;
}
Err(e) => {
failures += 1;
tracing::warn!(error = %e, engine = "anomaly", rule_id = %rule.id, "data fetch failed");
config_db
.update_anomaly_state(&rule.id, &rule.tenant_id, "no_data", &now_str, None)
Expand Down Expand Up @@ -391,6 +408,7 @@ async fn eval_anomaly_rules(
.update_anomaly_state(&rule.id, &rule.tenant_id, new_state, &now_str, None)
.await?;
}
observation.finish(EvaluationOutcome::Completed);
}

// Flush all per-series eval logs in one INSERT (previously one per series).
Expand All @@ -400,10 +418,22 @@ async fn eval_anomaly_rules(
eval_log_values.join(", ")
);
if let Err(e) = write_ch.query(&sql).execute().await {
failures += 1;
if let Some(metrics) = config_db.self_metrics() {
metrics.inc_counter(
"rush_engine_persistence_failures_total",
&[("engine", "anomaly_engine"), ("operation", "eval_log")],
1,
);
}
tracing::warn!(error = %e, engine = "anomaly", rows = eval_log_values.len(), "failed to write eval log batch");
}
}

anyhow::ensure!(
failures == 0,
"{failures} anomaly evaluation operation(s) failed"
);
Ok(())
}

Expand Down
8 changes: 6 additions & 2 deletions src/bin/anomaly_engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,10 @@ async fn main() -> anyhow::Result<()> {
std::env::var("RUSH_PROM_BASE_URL").unwrap_or_else(|_| "http://localhost:8080".to_string());

tracing::info!("wide-anomaly-engine starting");
// The standalone engine has no /metrics endpoint; give it a private registry so the
// engine-loop instrumentation still works (record_engine just updates in-memory atomics).
// The standalone worker exports the same bounded metrics as the API process.
let self_metrics = Arc::new(rush_api::self_metrics::SelfMetrics::new());
config_db.attach_metrics(self_metrics.clone());
rush_api::process_metrics::spawn(self_metrics.clone());
let query_limits = rush_api::query_governor::QueryGovernorConfig::at_startup(
config_db
.get_setting(rush_api::query_governor::QUERY_LIMITS_SETTING_KEY)
Expand All @@ -94,6 +95,8 @@ async fn main() -> anyhow::Result<()> {
// Takes the same lease as the in-process engine, so this deployment and any
// API replica can never both evaluate rules.
let shutdown = rush_api::shutdown::ShutdownController::new();
let _metrics_server =
rush_api::worker_metrics::spawn(self_metrics.clone(), shutdown.clone()).await?;
let election = rush_api::leader::LeaderElection::from_env(
config_db.client.clone(),
&rush_api::stats_engine::configured_instance_id(),
Expand All @@ -120,6 +123,7 @@ async fn main() -> anyhow::Result<()> {
}
}

shutdown.finish();
Ok(())
}

Expand Down
12 changes: 12 additions & 0 deletions src/clickhouse_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,15 @@ struct SavedQueryRow {
}

impl ConfigDb {
/// Attach the shared process registry before starting background work.
pub fn attach_metrics(&self, metrics: std::sync::Arc<crate::self_metrics::SelfMetrics>) {
let _ = self.metrics.set(metrics);
}

pub fn self_metrics(&self) -> Option<&crate::self_metrics::SelfMetrics> {
self.metrics.get().map(std::sync::Arc::as_ref)
}

pub async fn create_investigation_session(
&self,
id: &str,
Expand Down Expand Up @@ -2443,6 +2452,7 @@ mod sql_shape_tests {
}

pub struct ConfigDb {
metrics: std::sync::OnceLock<std::sync::Arc<crate::self_metrics::SelfMetrics>>,
pub client: Client,
/// Tenant name-or-id → flags. Negative results are cached too so unknown
/// tenants on the ingest path don't hammer ClickHouse.
Expand Down Expand Up @@ -2635,6 +2645,7 @@ impl ConfigDb {
.with_user(user)
.with_password(password);
let db = Self {
metrics: std::sync::OnceLock::new(),
client,
tenant_cache: DashMap::new(),
ingest_auth_cache: DashMap::new(),
Expand Down Expand Up @@ -12407,6 +12418,7 @@ mod session_tenant_regressions {

fn config(mock: &Mock) -> ConfigDb {
ConfigDb {
metrics: std::sync::OnceLock::new(),
client: Client::default()
.with_url(mock.url())
.with_compression(clickhouse::Compression::None),
Expand Down
130 changes: 130 additions & 0 deletions src/engine_metrics.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
//! Bounded evaluation outcomes, including cancelled work and successful no-data reads.
use crate::self_metrics::SelfMetrics;
use std::time::Instant;

#[derive(Clone, Copy)]
pub enum EvaluationOutcome {
Completed,
NoData,
Error,
AdmissionRejected,
InvalidConfig,
}

pub struct EvaluationObservation<'a> {
metrics: Option<&'a SelfMetrics>,
engine: &'static str,
started: Instant,
outcome: &'static str,
}

impl<'a> EvaluationObservation<'a> {
pub fn new(metrics: Option<&'a SelfMetrics>, engine: &'static str) -> Self {
let engine = match engine {
"monitor_engine" | "siem_engine" | "slo_engine" | "anomaly_engine" => engine,
_ => "other",
};
if let Some(metrics) = metrics {
metrics.add_gauge(
"rush_engine_evaluations_inflight",
&[("engine", engine)],
1.0,
);
}
Self {
metrics,
engine,
started: Instant::now(),
outcome: "cancelled",
}
}
pub fn finish(&mut self, outcome: EvaluationOutcome) {
self.outcome = match outcome {
EvaluationOutcome::Completed => "ok",
EvaluationOutcome::NoData => "no_data",
EvaluationOutcome::Error => "error",
EvaluationOutcome::AdmissionRejected => "admission_rejected",
EvaluationOutcome::InvalidConfig => "invalid_config",
};
}
}

impl Drop for EvaluationObservation<'_> {
fn drop(&mut self) {
if let Some(metrics) = self.metrics {
let labels = [("engine", self.engine), ("outcome", self.outcome)];
metrics.add_gauge(
"rush_engine_evaluations_inflight",
&[("engine", self.engine)],
-1.0,
);
metrics.inc_counter("rush_engine_evaluations_total", &labels, 1);
metrics.observe_histogram(
"rush_engine_evaluation_duration_ms",
&labels,
self.started.elapsed().as_secs_f64() * 1000.0,
);
if matches!(self.outcome, "ok" | "no_data") {
metrics.set_gauge(
"rush_engine_evaluation_last_success_timestamp_seconds",
&[("engine", self.engine)],
chrono::Utc::now().timestamp() as f64,
);
}
}
}
}

pub fn due(metrics: Option<&SelfMetrics>, engine: &'static str, count: usize) {
if let Some(metrics) = metrics {
metrics.set_gauge(
"rush_engine_evaluations_due",
&[("engine", engine)],
count as f64,
);
}
}

#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
#[tokio::test]
async fn cancellation_releases_inflight_and_is_not_a_success() {
let metrics = Arc::new(SelfMetrics::new());
let registry = metrics.clone();
let (tx, rx) = tokio::sync::oneshot::channel();
let task = tokio::spawn(async move {
let _observation = EvaluationObservation::new(Some(&registry), "monitor_engine");
tx.send(()).unwrap();
std::future::pending::<()>().await;
});
rx.await.unwrap();
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
let text = metrics.render_prometheus();
assert!(text.contains("rush_engine_evaluations_inflight{engine=\"monitor_engine\"} 0"));
assert!(text.contains(
"rush_engine_evaluations_total{engine=\"monitor_engine\",outcome=\"cancelled\"} 1"
));
assert!(!text.contains("last_success"));
}
#[test]
fn failed_work_does_not_advance_last_success_and_labels_are_bounded() {
let metrics = SelfMetrics::new();
{
let mut work = EvaluationObservation::new(Some(&metrics), "slo_engine");
work.finish(EvaluationOutcome::NoData);
}
{
let mut work = EvaluationObservation::new(Some(&metrics), "raw-rule-name");
work.finish(EvaluationOutcome::Error);
}
let text = metrics.render_prometheus();
assert!(text.contains(
"rush_engine_evaluation_last_success_timestamp_seconds{engine=\"slo_engine\"}"
));
assert!(!text.contains("last_success_timestamp_seconds{engine=\"other\"}"));
assert!(!text.contains("raw-rule-name"));
}
}
1 change: 1 addition & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ pub mod config;
pub mod cors;
pub mod detection_query;
pub mod edition;
pub mod engine_metrics;
pub mod eval_state;
pub mod github_repository_policy;
pub mod handlers;
Expand Down
1 change: 1 addition & 0 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2584,6 +2584,7 @@ async fn main() -> anyhow::Result<()> {
// outbox degradation is visible even if no one has scraped `/metrics` yet.
let self_metrics: std::sync::Arc<rush_api::self_metrics::SelfMetrics> =
std::sync::Arc::new(rush_api::self_metrics::SelfMetrics::new());
config_db.attach_metrics(self_metrics.clone());
let query_governor_config = QueryGovernorConfig::at_startup(
config_db
.get_setting(QUERY_LIMITS_SETTING_KEY)
Expand Down
28 changes: 23 additions & 5 deletions src/monitor_engine.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use crate::engine_metrics::{EvaluationObservation, EvaluationOutcome};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::sync::Arc;
use std::time::Instant;
Expand Down Expand Up @@ -342,8 +343,11 @@ async fn run_evaluation_cycle(
let now_str_ref = now_str.as_str();
let monitor_states_ref = &monitor_states;

let outcomes: Vec<(String, u64, bool)> =
crate::engine_metrics::due(config_db.self_metrics(), "monitor_engine", jobs.len());
let outcomes: Vec<(String, u64, bool, bool)> =
futures_util::stream::iter(jobs.into_iter().map(|(monitor, should_flush)| async move {
let mut observation =
EvaluationObservation::new(config_db.self_metrics(), "monitor_engine");
let result = match crate::query_governor::run_background(
&monitor.tenant_id,
evaluate_monitor(
Expand All @@ -361,9 +365,20 @@ async fn run_evaluation_cycle(
)
.await
{
Ok(result) => result,
Err(error) => Err(anyhow::anyhow!("background admission rejected: {error:?}")),
Ok(result) => {
observation.finish(if result.is_ok() {
EvaluationOutcome::Completed
} else {
EvaluationOutcome::Error
});
result
}
Err(error) => {
observation.finish(EvaluationOutcome::AdmissionRejected);
Err(anyhow::anyhow!("background admission rejected: {error:?}"))
}
};
let failed = result.is_err();
let (changes, persisted) = match result {
Ok(cp) => cp,
Err(e) => {
Expand All @@ -388,18 +403,21 @@ async fn run_evaluation_cycle(
.await
}
};
(monitor.id, changes, persisted)
(monitor.id, changes, persisted, failed)
}))
.buffer_unordered(ENGINE_CONCURRENCY)
.collect()
.await;

let mut state_changes: u64 = 0;
for (id, changes, persisted) in outcomes {
let mut failures = 0;
for (id, changes, persisted, failed) in outcomes {
failures += usize::from(failed);
state_changes += changes;
eval_state.record(id, now, persisted);
}

anyhow::ensure!(failures == 0, "{failures} monitor evaluation job(s) failed");
Ok((evaluated, state_changes))
}

Expand Down
Loading
Loading