From 1eaa4274831486a4fa3a38f4da5fe2b65ba1cd45 Mon Sep 17 00:00:00 2001 From: Mike Zupan Date: Sun, 15 Nov 2026 13:46:41 -0800 Subject: [PATCH] Expose replay health and count committed spool loss --- src/ch_writer.rs | 159 +++++++++++++++++++++++++++++++-- src/lib.rs | 1 + src/main.rs | 5 +- src/object_store_spool.rs | 53 ++++++++++- src/shutdown.rs | 5 ++ src/spool.rs | 180 ++++++++++++++++++++++++++++++++------ src/worker_metrics.rs | 103 ++++++++++++++++++++++ 7 files changed, 472 insertions(+), 34 deletions(-) create mode 100644 src/worker_metrics.rs diff --git a/src/ch_writer.rs b/src/ch_writer.rs index b55f0e7..b684209 100644 --- a/src/ch_writer.rs +++ b/src/ch_writer.rs @@ -25,6 +25,7 @@ use crate::models::ingest::{ use crate::models::profile::ProfileRow; use crate::models::rum::RumRecord; use crate::models::trace::WideEvent; +use crate::self_metrics::SelfMetrics; use crate::shutdown::{ShutdownController, ShutdownPhase}; use crate::spool::{IngestBuffer, SpoolFull}; @@ -371,6 +372,7 @@ impl BatchAccumulator { #[derive(Clone)] pub struct ChWriter { ch: Client, + metrics: Arc, /// Durable failure-path buffer. Backend-agnostic: `Disk` (default, no object /// store) or, later, `ObjectStore`. See `crate::spool::IngestBuffer`. pub buffer: Arc, @@ -391,6 +393,7 @@ impl ChWriter { pub fn with_batch_config(ch: Client, buffer: Arc, cfg: BatchConfig) -> Self { ChWriter { ch, + metrics: Arc::new(SelfMetrics::new()), buffer, firewall: Arc::new(std::sync::RwLock::new(Arc::new( crate::metric_firewall::MetricFirewall::default(), @@ -399,6 +402,12 @@ impl ChWriter { } } + /// Attach the process registry before cloning or starting background work. + pub fn with_metrics(mut self, metrics: Arc) -> Self { + self.metrics = metrics; + self + } + /// The active batching configuration. pub fn batch_config(&self) -> BatchConfig { self.batcher.cfg @@ -598,6 +607,43 @@ impl ChWriter { /// Spawn a background tokio task that replays spooled segments to CH. pub fn spawn_replayer(self, shutdown: ShutdownController) -> tokio::task::JoinHandle<()> { + self.buffer.attach_metrics(self.metrics.clone()); + let sampler = self.clone(); + let sampler_shutdown = shutdown.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(5)); + loop { + tokio::select! { + _ = sampler_shutdown.wait_for_finish() => break, + _ = interval.tick() => {} + } + let backend = sampler.buffer.backend_name(); + if sampler.buffer.refresh_counts().await.is_err() { + crate::spool::record_spool_error(&sampler.metrics, backend, "refresh"); + } + let labels = [("backend", backend)]; + for (name, value) in [ + ( + "rush_ingest_replay_pending_bytes", + sampler.buffer.total_bytes() as f64, + ), + ( + "rush_ingest_replay_pending_batches", + sampler.buffer.segment_count() as f64, + ), + ( + "rush_ingest_replay_oldest_age_seconds", + sampler.buffer.oldest_age_secs().await.unwrap_or(0) as f64, + ), + ( + "rush_ingest_replay_max_bytes", + sampler.buffer.max_bytes() as f64, + ), + ] { + sampler.metrics.set_gauge(name, &labels, value); + } + } + }); tokio::spawn(async move { let mut backoff = Duration::from_secs(5); const MAX_BACKOFF: Duration = Duration::from_secs(60); @@ -634,16 +680,20 @@ impl ChWriter { } }; + let started = Instant::now(); + let backend = self.buffer.backend_name(); let mut all_ok = true; + let mut partial = drain.discarded_bytes > 0; + let mut discarded_records = 0; + let mut discarded_bytes = 0; for (table, payload) in &drain.records { let batch: SpoolBatch = match decode_spool(payload) { Ok(b) => b, Err(e) => { - tracing::warn!( - error = %e, - table = table, - "replayer: deserialise failed — skipping record" - ); + discarded_records += 1; + discarded_bytes += payload.len() as u64; + partial = true; + tracing::warn!(error = %e, table = table, "replayer: deserialise failed — skipping record"); continue; } }; @@ -654,17 +704,73 @@ impl ChWriter { table = table, "replayer: CH insert failed — backing off" ); + self.metrics.inc_counter( + "rush_ingest_replay_records_total", + &[("backend", backend), ("outcome", "insert_error")], + 1, + ); all_ok = false; break; } + self.metrics.inc_counter( + "rush_ingest_replay_records_total", + &[("backend", backend), ("outcome", "inserted")], + 1, + ); } + self.metrics.observe_histogram( + "rush_ingest_replay_duration_ms", + &[("backend", backend)], + started.elapsed().as_secs_f64() * 1000.0, + ); if all_ok { let n = drain.records.len(); - self.buffer.commit(drain).await; + let committed = self.buffer.commit(drain).await; + if committed && discarded_records > 0 { + let labels = [ + ("backend", backend), + ("reason", "payload_decode"), + ("unit", "record"), + ]; + self.metrics.inc_counter( + "rush_ingest_spool_discarded_total", + &labels, + discarded_records, + ); + self.metrics.inc_counter( + "rush_ingest_spool_discarded_bytes_total", + &labels, + discarded_bytes, + ); + } + let outcome = if !committed { + "commit_error" + } else if partial { + "partial" + } else { + "committed" + }; + self.metrics.inc_counter( + "rush_ingest_replay_batches_total", + &[("backend", backend), ("outcome", outcome)], + 1, + ); + if committed && !partial { + self.metrics.set_gauge( + "rush_ingest_replay_last_success_timestamp_seconds", + &[("backend", backend)], + chrono::Utc::now().timestamp() as f64, + ); + } tracing::info!(records = n, "replayer: batch replayed and committed"); backoff = Duration::from_secs(5); // reset on success } else { + self.metrics.inc_counter( + "rush_ingest_replay_batches_total", + &[("backend", backend), ("outcome", "insert_error")], + 1, + ); // CH is still down — back off and stop this replay pass. // (drain not committed → retried next pass.) During graceful // shutdown this intentionally waits rather than exiting; @@ -947,6 +1053,47 @@ mod batch_tests { } } + #[tokio::test] + async fn invalid_replay_payload_is_partial_not_successful_delivery() { + let test = TestWriter::new("http://127.0.0.1:9", BatchConfig::default()); + // TestWriter's spool has a zero-byte cap; use a separate bounded spool. + let dir = test.1.join("replay"); + let buffer = Arc::new(IngestBuffer::Disk( + crate::spool::Spool::open(&dir, 1024).unwrap(), + )); + buffer + .append("untrusted-table-name", b"bad".to_vec()) + .await + .unwrap(); + let metrics = Arc::new(SelfMetrics::new()); + let writer = ChWriter::new( + Client::default().with_url("http://127.0.0.1:9"), + buffer.clone(), + ) + .with_metrics(metrics.clone()); + let shutdown = ShutdownController::new(); + shutdown.begin_drain(); + let task = writer.spawn_replayer(shutdown.clone()); + tokio::time::timeout(Duration::from_secs(2), async { + while buffer.segment_count() > 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + shutdown.finish(); + task.await.unwrap(); + let text = metrics.render_prometheus(); + assert!(text.contains("rush_ingest_spool_discarded_total{backend=\"disk\",reason=\"payload_decode\",unit=\"record\"} 1")); + assert!( + text.contains( + "rush_ingest_replay_batches_total{backend=\"disk\",outcome=\"partial\"} 1" + ) + ); + assert!(!text.contains("last_success_timestamp")); + assert!(!text.contains("untrusted-table-name")); + } + #[tokio::test] async fn rejected_retries_do_not_enter_the_buffer() { let test = TestWriter::new( diff --git a/src/lib.rs b/src/lib.rs index 0e3c529..265a874 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,6 +45,7 @@ pub mod spool; pub mod stats_engine; pub mod usage_accumulator; pub mod usage_tracker; +pub mod worker_metrics; use clickhouse::Client; use clickhouse::query::Query; diff --git a/src/main.rs b/src/main.rs index b1e60cd..e01666d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3120,7 +3120,7 @@ async fn main() -> anyhow::Result<()> { run_replayer, require_object_store, )?; - let writer = ChWriter::new(admin_ch.clone(), buffer); + let writer = ChWriter::new(admin_ch.clone(), buffer).with_metrics(self_metrics.clone()); let replayer_handle = if drain_only || run_replayer { Some(writer.clone().spawn_replayer(shutdown_controller.clone())) } else { @@ -3152,6 +3152,9 @@ async fn main() -> anyhow::Result<()> { // Drain-worker-only: this process exists solely to drain the ingest buffer // into ClickHouse. Don't serve HTTP, run engines, or load the firewall. if drain_only { + let _metrics_server = + rush_api::worker_metrics::spawn(self_metrics.clone(), shutdown_controller.clone()) + .await?; tracing::info!( backend = effective_backend, "RUSH_DRAIN_WORKER_ONLY — draining ingest buffer to ClickHouse; not serving HTTP" diff --git a/src/object_store_spool.rs b/src/object_store_spool.rs index 5a21cb5..3fff58d 100644 --- a/src/object_store_spool.rs +++ b/src/object_store_spool.rs @@ -10,9 +10,11 @@ //! Object body = `{table}\n{payload}` (table names have no newline; the payload //! is the same serde_json batch bytes the disk spool stores). +use crate::self_metrics::SelfMetrics; +use crate::spool::{record_spool_discard, record_spool_error}; use std::collections::BTreeMap; -use std::sync::Arc; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use bytes::Bytes; @@ -67,6 +69,7 @@ impl Drop for AppendReservation<'_> { } pub struct ObjectStoreSpool { + metrics: OnceLock>, store: Arc, prefix: String, max_bytes: u64, @@ -82,6 +85,15 @@ pub struct ObjectStoreSpool { } impl ObjectStoreSpool { + pub fn attach_metrics(&self, metrics: Arc) { + let _ = self.metrics.set(metrics); + } + fn record_error(&self, operation: &'static str) { + if let Some(metrics) = self.metrics.get() { + record_spool_error(metrics, "object_store", operation); + } + } + /// Build an S3/MinIO-backed buffer and seed counters from any objects left /// over from a previous run (durable across restarts/pods). pub async fn open_s3( @@ -123,6 +135,7 @@ impl ObjectStoreSpool { format!("{prefix}/") }; let s = ObjectStoreSpool { + metrics: OnceLock::new(), store, prefix, max_bytes, @@ -296,6 +309,7 @@ impl ObjectStoreSpool { .first_key_value() .map(|(key, size)| (key.clone(), *size)), Err(e) => { + self.record_error("list"); tracing::warn!(error = %e, "object-store buffer: list failed"); return None; } @@ -306,6 +320,7 @@ impl ObjectStoreSpool { Ok(r) => match r.bytes().await { Ok(b) => b, Err(e) => { + self.record_error("read"); tracing::warn!(error = %e, key = %key, "buffer: get bytes failed"); return None; } @@ -317,6 +332,7 @@ impl ObjectStoreSpool { continue; } Err(e) => { + self.record_error("get"); tracing::warn!(error = %e, key = %key, "buffer: get failed"); return None; } @@ -328,6 +344,15 @@ impl ObjectStoreSpool { if !self.delete_batch(&key, size).await { return None; } + if let Some(metrics) = self.metrics.get() { + record_spool_discard( + metrics, + "object_store", + "frame_decode", + "object", + size, + ); + } continue; // try the next object } } @@ -335,9 +360,12 @@ impl ObjectStoreSpool { } /// Delete a successfully-drained object. - pub async fn commit(&self, key: &OsPath, size: u64) { + pub async fn commit(&self, key: &OsPath, size: u64) -> bool { if self.delete_batch(key, size).await { self.committed.fetch_add(1, Ordering::Relaxed); + true + } else { + false } } @@ -345,6 +373,7 @@ impl ObjectStoreSpool { let _transaction = self.transactions.read().await; let mut index = self.index.lock().await; if let Err(e) = self.store.delete(key).await { + self.record_error("delete"); tracing::warn!(error = %e, key = %key, "object-store buffer: delete failed"); return false; } @@ -423,6 +452,26 @@ mod tests { } } + #[tokio::test] + async fn corrupt_objects_are_discarded_without_inflating_committed_batches() { + let store = store(); + store + .put( + &OsPath::from("ingest/0000000000000-00000000.batch"), + Bytes::from_static(b"broken").into(), + ) + .await + .unwrap(); + let spool = ObjectStoreSpool::open(store, "ingest", 1024).await.unwrap(); + let metrics = Arc::new(SelfMetrics::new()); + spool.attach_metrics(metrics.clone()); + assert!(spool.next_batch().await.is_none()); + assert!(spool.next_batch().await.is_none()); + assert_eq!(spool.committed_total(), 0); + assert_eq!(spool.total_bytes(), 0); + assert!(metrics.render_prometheus().contains("rush_ingest_spool_discarded_total{backend=\"object_store\",reason=\"frame_decode\",unit=\"object\"} 1")); + } + #[tokio::test] async fn drain_batches_keep_a_bounded_oldest_index_and_reuse_age_lookups() { let st = store(); diff --git a/src/shutdown.rs b/src/shutdown.rs index 3eac78a..a4c8d0e 100644 --- a/src/shutdown.rs +++ b/src/shutdown.rs @@ -55,6 +55,11 @@ impl ShutdownController { .await; } + pub async fn wait_for_finish(&self) { + self.wait_until(|phase| phase == ShutdownPhase::Finished) + .await; + } + pub async fn wait_for_drain(&self) { self.wait_until(|phase| phase >= ShutdownPhase::Draining) .await; diff --git a/src/spool.rs b/src/spool.rs index 3878370..045f14b 100644 --- a/src/spool.rs +++ b/src/spool.rs @@ -1,3 +1,4 @@ +use crate::self_metrics::SelfMetrics; /// Crash-safe segmented disk spool for ClickHouse write backpressure. /// /// Each segment is a flat file of framed records: @@ -7,14 +8,38 @@ /// The current (open) segment is rotated once it reaches SEGMENT_MAX_BYTES (32 MiB). /// On open, existing `*.spool` files are scanned to restore total_bytes. use std::fs::{self, File, OpenOptions}; -use std::io::{Read, Write}; +use std::io::{Read, Seek, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; const SEGMENT_MAX_BYTES: u64 = 32 * 1024 * 1024; // 32 MiB per segment +pub(crate) fn record_spool_discard( + metrics: &SelfMetrics, + backend: &'static str, + reason: &'static str, + unit: &'static str, + bytes: u64, +) { + let labels = [("backend", backend), ("reason", reason), ("unit", unit)]; + metrics.inc_counter("rush_ingest_spool_discarded_total", &labels, 1); + metrics.inc_counter("rush_ingest_spool_discarded_bytes_total", &labels, bytes); +} + +pub(crate) fn record_spool_error( + metrics: &SelfMetrics, + backend: &'static str, + operation: &'static str, +) { + metrics.inc_counter( + "rush_ingest_spool_errors_total", + &[("backend", backend), ("operation", operation)], + 1, + ); +} + /// Returned by `Spool::append` when the byte cap has been reached. #[derive(Debug)] pub struct SpoolFull; @@ -40,7 +65,13 @@ struct SpoolInner { segments: std::collections::BTreeSet, } +struct SegmentRead { + records: Vec<(String, Vec)>, + discarded_bytes: u64, +} + pub struct Spool { + metrics: OnceLock>, inner: Mutex, /// Monotonic count of segments successfully drained (for drain-rate metric). committed: AtomicU64, @@ -98,6 +129,7 @@ impl Spool { } Ok(Spool { + metrics: OnceLock::new(), inner: Mutex::new(SpoolInner { dir, max_bytes, @@ -201,6 +233,10 @@ impl Spool { /// encountered partway through a frame ends parsing and returns the records /// read so far, rather than discarding the whole (otherwise-valid) segment. pub fn read_segment(path: &Path) -> std::io::Result)>> { + Self::read_segment_with_loss(path).map(|read| read.records) + } + + fn read_segment_with_loss(path: &Path) -> std::io::Result { let metadata = fs::symlink_metadata(path)?; if metadata.file_type().is_symlink() || !metadata.is_file() { return Err(std::io::Error::new( @@ -214,38 +250,55 @@ impl Spool { let mut file = options.open(path)?; let mut records = Vec::new(); + let mut discarded_bytes = 0; loop { - // Read table name length. A clean EOF here is the normal end. + let frame_start = file.stream_position()?; + if frame_start == metadata.len() { + break; + } let mut buf4 = [0u8; 4]; - match file.read_exact(&mut buf4) { - Ok(_) => {} - Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break, - Err(e) => return Err(e), + if read_exact_or_eof(&mut file, &mut buf4)?.is_none() { + discarded_bytes = metadata.len().saturating_sub(frame_start); + break; } let tbl_len = u32::from_le_bytes(buf4) as usize; + // Bound corrupt lengths by the actual file before allocating. + if tbl_len as u64 > metadata.len().saturating_sub(file.stream_position()?) { + discarded_bytes = metadata.len().saturating_sub(frame_start); + break; + } let mut tbl_buf = vec![0u8; tbl_len]; if read_exact_or_eof(&mut file, &mut tbl_buf)?.is_none() { - break; // torn record — stop here + discarded_bytes = metadata.len().saturating_sub(frame_start); + break; } let table = match String::from_utf8(tbl_buf) { Ok(t) => t, - Err(_) => break, // garbage tail — stop + Err(_) => { + discarded_bytes = metadata.len().saturating_sub(frame_start); + break; + } }; - - // Read payload length + payload. if read_exact_or_eof(&mut file, &mut buf4)?.is_none() { + discarded_bytes = metadata.len().saturating_sub(frame_start); break; } let pay_len = u32::from_le_bytes(buf4) as usize; + if pay_len as u64 > metadata.len().saturating_sub(file.stream_position()?) { + discarded_bytes = metadata.len().saturating_sub(frame_start); + break; + } let mut payload = vec![0u8; pay_len]; if read_exact_or_eof(&mut file, &mut payload)?.is_none() { + discarded_bytes = metadata.len().saturating_sub(frame_start); break; } - records.push((table, payload)); } - - Ok(records) + Ok(SegmentRead { + records, + discarded_bytes, + }) } /// Seal (close + fsync) the currently open segment so it becomes eligible @@ -262,13 +315,20 @@ impl Spool { } /// Remove a segment file and update total_bytes + the in-memory index. - pub fn remove_segment(&self, path: &Path) { + pub fn remove_segment(&self, path: &Path) -> bool { let size = fs::metadata(path).map(|m| m.len()).unwrap_or(0); - let _ = fs::remove_file(path); + if let Err(error) = fs::remove_file(path) + && error.kind() != std::io::ErrorKind::NotFound + { + if let Some(metrics) = self.metrics.get() { + record_spool_error(metrics, "disk", "delete"); + } + return false; + } let mut g = self.inner.lock().unwrap(); g.total_bytes = g.total_bytes.saturating_sub(size); g.segments.remove(path); - self.committed.fetch_add(1, Ordering::Relaxed); + true } pub fn total_bytes(&self) -> u64 { @@ -344,6 +404,7 @@ fn read_exact_or_eof(file: &mut File, buf: &mut [u8]) -> std::io::Result)>, + pub discarded_bytes: u64, handle: BatchHandle, } @@ -361,6 +422,15 @@ pub enum IngestBuffer { } impl IngestBuffer { + pub fn attach_metrics(&self, metrics: Arc) { + match self { + Self::Disk(spool) => { + let _ = spool.metrics.set(metrics); + } + Self::ObjectStore(spool) => spool.attach_metrics(metrics), + } + } + /// Append a framed record (failure-path hot call). 429 when over the cap. /// Disk appends are blocking file writes serialized by the spool mutex, so /// they run on the blocking pool — otherwise, exactly when ClickHouse is @@ -462,11 +532,17 @@ impl IngestBuffer { } } }; - match Spool::read_segment(&path) { - Ok(records) => return Some(DrainBatch { records, handle: BatchHandle::Disk(path) }), + match Spool::read_segment_with_loss(&path) { + Ok(read) => { + return Some(DrainBatch { records: read.records, discarded_bytes: read.discarded_bytes, handle: BatchHandle::Disk(path) }); + }, Err(e) => { tracing::error!(error = %e, segment = %path.display(), "ingest buffer: failed to read segment — discarding"); - s.remove_segment(&path); + let bytes = fs::metadata(&path).map(|m| m.len()).unwrap_or(0); + if !s.remove_segment(&path) { return None; } + if let Some(metrics) = s.metrics.get() { + record_spool_discard(metrics, "disk", "segment_read", "segment", bytes); + } continue; } } @@ -479,6 +555,7 @@ impl IngestBuffer { IngestBuffer::ObjectStore(s) => { s.next_batch().await.map(|(key, size, records)| DrainBatch { records, + discarded_bytes: 0, handle: BatchHandle::ObjectStore { key, size }, }) } @@ -486,14 +563,35 @@ impl IngestBuffer { } /// Commit (remove/ack) a batch that was successfully drained to ClickHouse. - pub async fn commit(&self, batch: DrainBatch) { + pub async fn commit(&self, batch: DrainBatch) -> bool { match (self, batch.handle) { - (IngestBuffer::Disk(s), BatchHandle::Disk(path)) => s.remove_segment(&path), + (IngestBuffer::Disk(s), BatchHandle::Disk(path)) => { + if s.remove_segment(&path) { + if batch.discarded_bytes > 0 + && let Some(metrics) = s.metrics.get() + { + record_spool_discard( + metrics, + "disk", + "frame_decode", + "tail", + batch.discarded_bytes, + ); + } + s.committed.fetch_add(1, Ordering::Relaxed); + true + } else { + false + } + } (IngestBuffer::ObjectStore(s), BatchHandle::ObjectStore { key, size }) => { s.commit(&key, size).await } // Mismatched handle/backend can't happen (handles are minted by the same backend). - _ => tracing::error!("ingest buffer: commit handle/backend mismatch"), + _ => { + tracing::error!("ingest buffer: commit handle/backend mismatch"); + false + } } } } @@ -525,13 +623,45 @@ fn sync_file(f: &mut File) -> std::io::Result<()> { #[cfg(test)] mod tests { - use super::Spool; - use std::fs; + use super::{IngestBuffer, SelfMetrics, Spool}; + use std::fs::{self, OpenOptions}; + use std::io::Write; + use std::sync::Arc; fn temp_path(label: &str) -> std::path::PathBuf { std::env::temp_dir().join(format!("rush-spool-{label}-{}", uuid::Uuid::new_v4())) } + #[tokio::test] + async fn torn_tail_is_counted_only_when_committed_not_on_read_retries() { + let path = temp_path("torn-tail-metrics"); + let spool = Spool::open(&path, 1024 * 1024).unwrap(); + spool.append("logs", b"accepted").unwrap(); + spool.seal_current(); + let segment = spool.take_oldest_segment().unwrap(); + OpenOptions::new() + .append(true) + .open(&segment) + .unwrap() + .write_all(&[1, 2]) + .unwrap(); + let buffer = Arc::new(IngestBuffer::Disk(spool)); + let metrics = Arc::new(SelfMetrics::new()); + buffer.attach_metrics(metrics.clone()); + let first = buffer.next_batch().await.unwrap(); + assert_eq!(first.records.len(), 1); + assert_eq!(first.discarded_bytes, 2); + drop(first); // insert failure leaves the segment available + let retry = buffer.next_batch().await.unwrap(); + assert!(!metrics.render_prometheus().contains("discarded_total")); + assert!(buffer.commit(retry).await); + assert_eq!(buffer.committed_total(), 1); + let text = metrics.render_prometheus(); + assert!(text.contains("rush_ingest_spool_discarded_total{backend=\"disk\",reason=\"frame_decode\",unit=\"tail\"} 1")); + assert!(text.contains("rush_ingest_spool_discarded_bytes_total{backend=\"disk\",reason=\"frame_decode\",unit=\"tail\"} 2")); + fs::remove_dir_all(path).unwrap(); + } + #[test] #[cfg(unix)] fn spool_migrates_directory_and_segment_permissions() { diff --git a/src/worker_metrics.rs b/src/worker_metrics.rs new file mode 100644 index 0000000..7f0d6e9 --- /dev/null +++ b/src/worker_metrics.rs @@ -0,0 +1,103 @@ +//! A small scrape listener for worker processes that do not serve the API. +use crate::{self_metrics::SelfMetrics, shutdown::ShutdownController}; +use axum::{Router, extract::State, routing::get}; +use std::sync::Arc; + +pub const DEFAULT_WORKER_METRICS_ADDR: &str = "0.0.0.0:9090"; + +/// An explicit empty address disables the listener. Bind errors are startup +/// errors, rather than silently running a worker without its configured metrics. +pub async fn spawn( + metrics: Arc, + shutdown: ShutdownController, +) -> anyhow::Result>> { + let address = std::env::var("RUSH_WORKER_METRICS_ADDR") + .unwrap_or_else(|_| DEFAULT_WORKER_METRICS_ADDR.to_string()); + if address.trim().is_empty() { + return Ok(None); + } + let listener = tokio::net::TcpListener::bind(address.trim()).await?; + Ok(Some(spawn_listener(listener, metrics, shutdown))) +} + +fn spawn_listener( + listener: tokio::net::TcpListener, + metrics: Arc, + shutdown: ShutdownController, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + if let Err(error) = axum::serve(listener, router(metrics)) + .with_graceful_shutdown(async move { shutdown.wait_for_finish().await }) + .await + { + tracing::error!(%error, "worker metrics listener failed"); + } + }) +} + +fn router(metrics: Arc) -> Router { + Router::new() + .route( + "/metrics", + get(|State(metrics): State>| async move { + ( + [("content-type", "text/plain; version=0.0.4; charset=utf-8")], + metrics.render_prometheus(), + ) + }), + ) + .with_state(metrics) +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn worker_exposes_metrics_without_serving_api_routes() { + let metrics = Arc::new(SelfMetrics::new()); + metrics.inc_counter( + "rush_ingest_replay_batches_total", + &[("outcome", "committed")], + 1, + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let shutdown = ShutdownController::new(); + let server = spawn_listener(listener, metrics, shutdown.clone()); + shutdown.request(); // metrics remain available while the worker drains + + let client = reqwest::Client::new(); + let response = client + .get(format!("http://{address}/metrics")) + .send() + .await + .unwrap(); + assert!( + response.headers()["content-type"] + .to_str() + .unwrap() + .starts_with("text/plain") + ); + assert!( + response + .text() + .await + .unwrap() + .contains("rush_ingest_replay_batches_total") + ); + assert_eq!( + client + .get(format!("http://{address}/api/v1/tenants")) + .send() + .await + .unwrap() + .status(), + 404 + ); + shutdown.finish(); + tokio::time::timeout(std::time::Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); + } +}