diff --git a/src/alert_engine.rs b/src/alert_engine.rs index 32fe59a..77e0114 100644 --- a/src/alert_engine.rs +++ b/src/alert_engine.rs @@ -348,9 +348,28 @@ pub async fn notify_channels( ) { for channel_id in channel_ids { let channel = match config_db.get_channel(channel_id, tenant_id).await { - Ok(Some(channel)) if channel.enabled => channel, - Ok(_) => continue, + Ok(Some(channel)) if channel.enabled => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + notification.source, + "ready", + ); + channel + } + Ok(_) => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + notification.source, + "skipped", + ); + continue; + } Err(error) => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + notification.source, + "error", + ); tracing::warn!(source = notification.source, %channel_id, %error, "failed to load notification channel"); continue; } @@ -360,22 +379,27 @@ pub async fn notify_channels( } else { notification.alert_id }; - let result = send_channel_notification_with_fields( - &channel, - notification.message, - notification.alert_name, - notification.alert_state, - notification.value, - notification.threshold, - notification.signal_type, - notification.condition_op, - notification.description, - notification_id, - notification.runbook_url, - notification.webhook_fields, - http_client, - smtp_config, - smtp_transport, + let result = crate::notification_metrics::observe_delivery( + config_db.self_metrics(), + notification.source, + &channel.channel_type, + send_channel_notification_with_fields( + &channel, + notification.message, + notification.alert_name, + notification.alert_state, + notification.value, + notification.threshold, + notification.signal_type, + notification.condition_op, + notification.description, + notification_id, + notification.runbook_url, + notification.webhook_fields, + http_client, + smtp_config, + smtp_transport, + ), ) .await; let (status, error_msg) = match &result { @@ -385,7 +409,7 @@ pub async fn notify_channels( ("failed", error.clone()) } }; - let _ = config_db + let logged = config_db .create_notification_log( channel_id, tenant_id, @@ -396,6 +420,11 @@ pub async fn notify_channels( &error_msg, ) .await; + crate::notification_metrics::record_log_write( + config_db.self_metrics(), + notification.source, + logged.is_ok(), + ); } } @@ -899,25 +928,53 @@ async fn eval_alert_rule( serde_json::from_str(&rule.notification_channel_ids).unwrap_or_default(); let alert_state_str = if triggered { "FIRING" } else { "RESOLVED" }; for channel_id in &channel_ids { - if let Ok(Some(channel)) = config_db.get_channel_by_id(channel_id).await { - if !channel.enabled { + let channel = match config_db.get_channel_by_id(channel_id).await { + Ok(Some(channel)) if channel.enabled => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + "alert_rule", + "ready", + ); + channel + } + Ok(_) => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + "alert_rule", + "skipped", + ); continue; } - let result = send_channel_notification( - &channel, - &message, - &rule.name, - alert_state_str, - value, - threshold, - &rule.signal_type, - &rule.condition_op, - &rule.description, - &rule.id, - &rule.runbook_url, - http_client, - smtp_config, - smtp_transport, + Err(_) => { + crate::notification_metrics::record_lookup( + config_db.self_metrics(), + "alert_rule", + "error", + ); + continue; + } + }; + { + let result = crate::notification_metrics::observe_delivery( + config_db.self_metrics(), + "alert_rule", + &channel.channel_type, + send_channel_notification( + &channel, + &message, + &rule.name, + alert_state_str, + value, + threshold, + &rule.signal_type, + &rule.condition_op, + &rule.description, + &rule.id, + &rule.runbook_url, + http_client, + smtp_config, + smtp_transport, + ), ) .await; @@ -933,7 +990,7 @@ async fn eval_alert_rule( } }; - let _ = config_db + let logged = config_db .create_notification_log( channel_id, &channel.tenant_id, @@ -944,6 +1001,11 @@ async fn eval_alert_rule( &error_msg, ) .await; + crate::notification_metrics::record_log_write( + config_db.self_metrics(), + "alert_rule", + logged.is_ok(), + ); } } diff --git a/src/handlers/alerts.rs b/src/handlers/alerts.rs index 25d909a..09cc4df 100644 --- a/src/handlers/alerts.rs +++ b/src/handlers/alerts.rs @@ -465,21 +465,26 @@ pub async fn test_channel( }; let http_client = reqwest::Client::new(); - let result = crate::alert_engine::send_channel_notification( - &channel, - &test_message, - "Test Alert", - "TEST", - 0.0, - 0.0, - "", - "", - "This is a test notification from Rush Observability.", - "", - "", - &http_client, - &smtp_config, - &smtp_transport, + let result = crate::notification_metrics::observe_delivery( + Some(&state.self_metrics), + "test", + &channel.channel_type, + crate::alert_engine::send_channel_notification( + &channel, + &test_message, + "Test Alert", + "TEST", + 0.0, + 0.0, + "", + "", + "This is a test notification from Rush Observability.", + "", + "", + &http_client, + &smtp_config, + &smtp_transport, + ), ) .await; @@ -501,7 +506,7 @@ pub async fn test_channel( ) .await; - let _ = state + let logged = state .config_db .create_notification_log( &id, @@ -513,6 +518,11 @@ pub async fn test_channel( &error_msg, ) .await; + crate::notification_metrics::record_log_write( + Some(&state.self_metrics), + "test", + logged.is_ok(), + ); match result { Ok(()) => Ok(Json( diff --git a/src/lib.rs b/src/lib.rs index 9d195b7..d015948 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,7 @@ pub mod metric_firewall; pub mod migrations; pub mod models; pub mod monitor_engine; +pub mod notification_metrics; pub mod object_store_spool; pub mod outbound; pub mod pagination; diff --git a/src/notification_metrics.rs b/src/notification_metrics.rs new file mode 100644 index 0000000..7d97a43 --- /dev/null +++ b/src/notification_metrics.rs @@ -0,0 +1,161 @@ +//! Notification health without destination, tenant, alert or credential labels. +use crate::self_metrics::SelfMetrics; +use std::{future::Future, time::Instant}; + +fn source(value: &str) -> &'static str { + match value { + "monitor" => "monitor", + "slo" => "slo", + "anomaly" => "anomaly", + "detection" => "detection", + "alert_rule" => "alert_rule", + "test" => "test", + _ => "other", + } +} +fn channel_type(value: &str) -> &'static str { + match value { + "email" => "email", + "slack" => "slack", + "slack_app" => "slack_app", + "webhook" => "webhook", + "pagerduty" => "pagerduty", + "rootly" => "rootly", + "discord" => "discord", + "alertmanager" => "alertmanager", + _ => "other", + } +} +struct DeliveryObservation<'a> { + metrics: Option<&'a SelfMetrics>, + source: &'static str, + channel: &'static str, + started: Instant, + outcome: &'static str, +} +impl Drop for DeliveryObservation<'_> { + fn drop(&mut self) { + if let Some(metrics) = self.metrics { + let labels = [("source", self.source), ("channel_type", self.channel)]; + metrics.add_gauge("rush_notification_deliveries_inflight", &labels, -1.0); + let labels = [ + ("source", self.source), + ("channel_type", self.channel), + ("outcome", self.outcome), + ]; + metrics.inc_counter("rush_notification_deliveries_total", &labels, 1); + metrics.observe_histogram( + "rush_notification_delivery_duration_ms", + &labels, + self.started.elapsed().as_secs_f64() * 1000.0, + ); + } + } +} + +pub async fn observe_delivery( + metrics: Option<&SelfMetrics>, + origin: &str, + channel: &str, + work: impl Future>, +) -> Result<(), String> { + let mut observation = DeliveryObservation { + metrics, + source: source(origin), + channel: channel_type(channel), + started: Instant::now(), + outcome: "cancelled", + }; + if let Some(metrics) = metrics { + metrics.add_gauge( + "rush_notification_deliveries_inflight", + &[ + ("source", observation.source), + ("channel_type", observation.channel), + ], + 1.0, + ); + } + let result = work.await; + observation.outcome = if result.is_ok() { "sent" } else { "error" }; + result +} + +pub fn record_lookup(metrics: Option<&SelfMetrics>, origin: &str, outcome: &'static str) { + if let Some(metrics) = metrics { + let outcome = match outcome { + "ready" | "skipped" | "error" => outcome, + _ => "other", + }; + metrics.inc_counter( + "rush_notification_channel_lookups_total", + &[("source", source(origin)), ("outcome", outcome)], + 1, + ); + } +} +pub fn record_log_write(metrics: Option<&SelfMetrics>, origin: &str, ok: bool) { + if let Some(metrics) = metrics { + metrics.inc_counter( + "rush_notification_log_writes_total", + &[ + ("source", source(origin)), + ("outcome", if ok { "ok" } else { "error" }), + ], + 1, + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + #[tokio::test] + async fn delivery_outcomes_are_distinct_and_labels_never_include_error_or_destination() { + let metrics = SelfMetrics::new(); + observe_delivery(Some(&metrics), "monitor", "email", async { Ok(()) }) + .await + .unwrap(); + assert!( + observe_delivery( + Some(&metrics), + "tenant-secret", + "https://secret-destination", + async { Err("credential-secret".into()) } + ) + .await + .is_err() + ); + record_log_write(Some(&metrics), "monitor", false); + let text = metrics.render_prometheus(); + assert!(text.contains("rush_notification_deliveries_total{channel_type=\"email\",outcome=\"sent\",source=\"monitor\"} 1")); + assert!(text.contains("rush_notification_deliveries_total{channel_type=\"other\",outcome=\"error\",source=\"other\"} 1")); + assert!(text.contains( + "rush_notification_log_writes_total{outcome=\"error\",source=\"monitor\"} 1" + )); + assert!(!text.contains("secret")); + } + #[tokio::test] + async fn cancelled_delivery_releases_inflight_without_recording_a_send() { + let metrics = Arc::new(SelfMetrics::new()); + let registry = metrics.clone(); + let (tx, rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + observe_delivery(Some(®istry), "slo", "webhook", async { + tx.send(()).unwrap(); + std::future::pending::>().await + }) + .await + }); + rx.await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + let text = metrics.render_prometheus(); + assert!(text.contains( + "rush_notification_deliveries_inflight{channel_type=\"webhook\",source=\"slo\"} 0" + )); + assert!(text.contains("rush_notification_deliveries_total{channel_type=\"webhook\",outcome=\"cancelled\",source=\"slo\"} 1")); + assert!(!text.contains("outcome=\"sent\"")); + } +}