Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
222 changes: 215 additions & 7 deletions src/ch_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
/// Replay: `spawn_replayer` consumes the oldest segment every ~5 s,
/// retrying with exponential back-off up to 60 s.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::{Duration, Instant};

use clickhouse::Client;
Expand All @@ -25,6 +26,7 @@ use crate::models::ingest::{
use crate::models::profile::ProfileRow;
use crate::models::rum::RumRecord;
use crate::models::trace::WideEvent;
use crate::self_metrics::SelfMetrics;
use crate::shutdown::{ShutdownController, ShutdownPhase};
use crate::spool::{IngestBuffer, SpoolFull};

Expand Down Expand Up @@ -336,19 +338,74 @@ struct TableBuffer {
needs_flush: bool,
}

const BATCH_TABLES: [&str; SpoolBatch::SLOTS] = [
"spans_raw",
"spans",
"logs",
"metrics_gauge",
"metrics_sum",
"rum",
"rum_replay",
"metrics_histogram",
"metrics_exp_histogram",
"metrics_summary",
"profile_samples",
];

#[derive(Default)]
struct BatchHealth {
rows: AtomicU64,
bytes: AtomicU64,
first_row_ms: AtomicU64,
blocked: AtomicBool,
}

fn unix_millis() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64
}

/// Records a cancelled attempt too: cancellation must not look like delivery.
struct DeliveryObservation<'a> {
metrics: &'a SelfMetrics,
table: &'static str,
rows: u64,
started: Instant,
outcome: &'static str,
}

impl Drop for DeliveryObservation<'_> {
fn drop(&mut self) {
let labels = [("table", self.table), ("outcome", self.outcome)];
self.metrics
.inc_counter("rush_ingest_delivery_total", &labels, 1);
self.metrics
.inc_counter("rush_ingest_delivery_rows_total", &labels, self.rows);
self.metrics.observe_histogram(
"rush_ingest_delivery_duration_ms",
&labels,
self.started.elapsed().as_secs_f64() * 1000.0,
);
}
}

/// The in-process batching layer: one buffer per ClickHouse table, each behind
/// its own async mutex so independent ingest tasks contend only with same-table
/// peers and the flusher.
pub struct BatchAccumulator {
cfg: BatchConfig,
slots: [AsyncMutex<TableBuffer>; SpoolBatch::SLOTS],
health: [BatchHealth; SpoolBatch::SLOTS],
}

impl BatchAccumulator {
fn new(cfg: BatchConfig) -> Self {
BatchAccumulator {
cfg,
slots: std::array::from_fn(|_| AsyncMutex::new(TableBuffer::default())),
health: std::array::from_fn(|_| BatchHealth::default()),
}
}

Expand All @@ -371,6 +428,7 @@ impl BatchAccumulator {
#[derive(Clone)]
pub struct ChWriter {
ch: Client,
metrics: Arc<SelfMetrics>,
/// Durable failure-path buffer. Backend-agnostic: `Disk` (default, no object
/// store) or, later, `ObjectStore`. See `crate::spool::IngestBuffer`.
pub buffer: Arc<IngestBuffer>,
Expand All @@ -391,6 +449,7 @@ impl ChWriter {
pub fn with_batch_config(ch: Client, buffer: Arc<IngestBuffer>, cfg: BatchConfig) -> Self {
ChWriter {
ch,
metrics: Arc::new(SelfMetrics::new()),
buffer,
firewall: Arc::new(std::sync::RwLock::new(Arc::new(
crate::metric_firewall::MetricFirewall::default(),
Expand All @@ -399,6 +458,42 @@ impl ChWriter {
}
}

/// Attach the process registry before cloning or starting background work.
pub fn with_metrics(mut self, metrics: Arc<SelfMetrics>) -> Self {
self.metrics = metrics;
self
}

fn record_pending(&self, index: usize) {
let health = &self.batcher.health[index];
let labels = [("table", BATCH_TABLES[index])];
let first = health.first_row_ms.load(Ordering::Relaxed);
for (name, value) in [
(
"rush_ingest_batch_pending_rows",
health.rows.load(Ordering::Relaxed) as f64,
),
(
"rush_ingest_batch_pending_bytes",
health.bytes.load(Ordering::Relaxed) as f64,
),
(
"rush_ingest_batch_oldest_age_seconds",
if first == 0 {
0.0
} else {
unix_millis().saturating_sub(first) as f64 / 1000.0
},
),
(
"rush_ingest_batch_flush_blocked",
u8::from(health.blocked.load(Ordering::Relaxed)) as f64,
),
] {
self.metrics.set_gauge(name, &labels, value);
}
}

/// The active batching configuration.
pub fn batch_config(&self) -> BatchConfig {
self.batcher.cfg
Expand Down Expand Up @@ -459,9 +554,15 @@ impl ChWriter {
let bytes = batch_serialized_size(&batch)?;
// Do not turn pending mutex waiters into an unbounded second queue of
// request bodies while ClickHouse is slow or unresponsive.
let mut slot = self.batcher.slots[batch.slot()]
.try_lock()
.map_err(|_| WriteError::Backpressure)?;
let index = batch.slot();
let mut slot = self.batcher.slots[index].try_lock().map_err(|_| {
self.metrics.inc_counter(
"rush_ingest_batch_rejections_total",
&[("table", batch.table()), ("reason", "busy")],
1,
);
WriteError::Backpressure
})?;
let rows = slot.batch.as_ref().map_or(0, SpoolBatch::len);
if slot.needs_flush
|| rows.saturating_add(batch.len()) >= self.batcher.cfg.max_rows
Expand All @@ -482,6 +583,19 @@ impl ChWriter {
}
}
slot.bytes = slot.bytes.saturating_add(bytes);
let health = &self.batcher.health[index];
health.rows.store(
slot.batch.as_ref().map_or(0, SpoolBatch::len) as u64,
Ordering::Relaxed,
);
health.bytes.store(slot.bytes as u64, Ordering::Relaxed);
let _ = health.first_row_ms.compare_exchange(
0,
unix_millis(),
Ordering::Relaxed,
Ordering::Relaxed,
);
// The independent sampler publishes these atomics off the ingest hot path.
Ok(())
}

Expand All @@ -490,7 +604,16 @@ impl ChWriter {
async fn flush_slot(&self, slot: &mut TableBuffer) -> Result<(), WriteError> {
if let Some(batch) = &slot.batch {
slot.needs_flush = true;
let index = batch.slot();
let health = &self.batcher.health[index];
health.blocked.store(true, Ordering::Relaxed);
self.record_pending(index);
self.write_now(batch).await?;
health.rows.store(0, Ordering::Relaxed);
health.bytes.store(0, Ordering::Relaxed);
health.first_row_ms.store(0, Ordering::Relaxed);
health.blocked.store(false, Ordering::Relaxed);
self.record_pending(index);
}
*slot = TableBuffer::default();
Ok(())
Expand All @@ -507,9 +630,24 @@ impl ChWriter {
let row_count = batch.len();
let table = batch.table();

let mut observation = DeliveryObservation {
metrics: &self.metrics,
table,
rows: row_count as u64,
started: Instant::now(),
outcome: "cancelled",
};
match try_insert(&self.ch, batch).await {
Ok(()) => Ok(()),
Ok(()) => {
observation.outcome = "inserted";
Ok(())
}
Err(e) => {
self.metrics.inc_counter(
"rush_ingest_insert_failures_total",
&[("table", table)],
1,
);
tracing::warn!(
error = %e,
table = table,
Expand All @@ -518,11 +656,19 @@ impl ChWriter {
);

// Serialise for the spool (zstd-compressed MessagePack).
let payload = encode_spool(batch)?;
let payload = encode_spool(batch).inspect_err(|_| {
observation.outcome = "encode_error";
})?;

match self.buffer.append(table, payload).await {
Ok(()) => Ok(()),
Err(SpoolFull) => Err(WriteError::Backpressure),
Ok(()) => {
observation.outcome = "spooled";
Ok(())
}
Err(SpoolFull) => {
observation.outcome = "backpressure";
Err(WriteError::Backpressure)
}
}
}
}
Expand Down Expand Up @@ -556,6 +702,19 @@ impl ChWriter {
if self.batcher.cfg.disabled() {
return;
}
// A separate sampler uses atomics, so a network-stalled flush cannot
// freeze the age or hide the accepted rows behind the slot mutex.
let sampler = self.clone();
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_millis(100));
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
interval.tick().await;
for index in 0..SpoolBatch::SLOTS {
sampler.record_pending(index);
}
}
});
let me = self.clone();
let tick = me
.batcher
Expand Down Expand Up @@ -947,6 +1106,50 @@ mod batch_tests {
}
}

#[tokio::test]
async fn pending_and_delivery_metrics_distinguish_acceptance_failure_and_recovery() {
use clickhouse::test::{Mock, handlers};
let mock = Mock::new();
let test = TestWriter::new(
mock.url(),
BatchConfig {
max_rows: 3,
..BatchConfig::default()
},
);
test.0.write(gauge(2)).await.unwrap();
test.0.batcher.health[gauge(1).slot()]
.first_row_ms
.store(unix_millis() - 2_000, Ordering::Relaxed);
test.0.record_pending(gauge(1).slot());
let before = test.0.metrics.render_prometheus();
assert!(before.contains("rush_ingest_batch_pending_rows{table=\"metrics_gauge\"} 2"));
assert!(!before.contains("rush_ingest_delivery_total"));
mock.add(handlers::failure(
axum::http::StatusCode::SERVICE_UNAVAILABLE,
));
assert!(matches!(
test.0.write(gauge(1)).await,
Err(WriteError::Backpressure)
));
let failed = test.0.metrics.render_prometheus();
assert!(failed.contains("rush_ingest_batch_flush_blocked{table=\"metrics_gauge\"} 1"));
assert!(failed.contains(
"rush_ingest_delivery_rows_total{outcome=\"backpressure\",table=\"metrics_gauge\"} 2"
));
let received = mock.add(handlers::record::<GaugeRow>());
test.0.flush_all().await;
assert_eq!(received.collect::<Vec<_>>().await.len(), 2);
let recovered = test.0.metrics.render_prometheus();
assert!(recovered.contains("rush_ingest_batch_pending_rows{table=\"metrics_gauge\"} 0"));
assert!(
recovered.contains("rush_ingest_batch_oldest_age_seconds{table=\"metrics_gauge\"} 0")
);
assert!(recovered.contains(
"rush_ingest_delivery_rows_total{outcome=\"inserted\",table=\"metrics_gauge\"} 2"
));
}

#[tokio::test]
async fn rejected_retries_do_not_enter_the_buffer() {
let test = TestWriter::new(
Expand Down Expand Up @@ -1112,6 +1315,11 @@ mod batch_tests {
let slot = test.0.batcher.slots[gauge(1).slot()].lock().await;
assert_eq!(slot.batch.as_ref().unwrap().len(), 1);
assert!(slot.needs_flush);
let metrics = test.0.metrics.render_prometheus();
assert!(metrics.contains(
"rush_ingest_delivery_total{outcome=\"cancelled\",table=\"metrics_gauge\"} 1"
));
assert!(metrics.contains("rush_ingest_batch_pending_rows{table=\"metrics_gauge\"} 1"));
server.abort();
}
}
2 changes: 1 addition & 1 deletion src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3120,7 +3120,7 @@ async fn main() -> anyhow::Result<()> {
run_replayer,
require_object_store,
)?;
let writer = ChWriter::new(admin_ch.clone(), buffer);
let writer = ChWriter::new(admin_ch.clone(), buffer).with_metrics(self_metrics.clone());
let replayer_handle = if drain_only || run_replayer {
Some(writer.clone().spawn_replayer(shutdown_controller.clone()))
} else {
Expand Down
Loading