Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 144 additions & 6 deletions src/ch_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,43 @@ impl ChWriter {

/// Spawn a background tokio task that replays spooled segments to CH.
pub fn spawn_replayer(self, shutdown: ShutdownController) -> tokio::task::JoinHandle<()> {
self.buffer.attach_metrics(self.metrics.clone());
let sampler = self.clone();
let sampler_shutdown = shutdown.clone();
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
tokio::select! {
_ = sampler_shutdown.wait_for_finish() => break,
_ = interval.tick() => {}
}
let backend = sampler.buffer.backend_name();
if sampler.buffer.refresh_counts().await.is_err() {
crate::spool::record_spool_error(&sampler.metrics, backend, "refresh");
}
let labels = [("backend", backend)];
for (name, value) in [
(
"rush_ingest_replay_pending_bytes",
sampler.buffer.total_bytes() as f64,
),
(
"rush_ingest_replay_pending_batches",
sampler.buffer.segment_count() as f64,
),
(
"rush_ingest_replay_oldest_age_seconds",
sampler.buffer.oldest_age_secs().await.unwrap_or(0) as f64,
),
(
"rush_ingest_replay_max_bytes",
sampler.buffer.max_bytes() as f64,
),
] {
sampler.metrics.set_gauge(name, &labels, value);
}
}
});
tokio::spawn(async move {
let mut backoff = Duration::from_secs(5);
const MAX_BACKOFF: Duration = Duration::from_secs(60);
Expand Down Expand Up @@ -793,16 +830,20 @@ impl ChWriter {
}
};

let started = Instant::now();
let backend = self.buffer.backend_name();
let mut all_ok = true;
let mut partial = drain.discarded_bytes > 0;
let mut discarded_records = 0;
let mut discarded_bytes = 0;
for (table, payload) in &drain.records {
let batch: SpoolBatch = match decode_spool(payload) {
Ok(b) => b,
Err(e) => {
tracing::warn!(
error = %e,
table = table,
"replayer: deserialise failed — skipping record"
);
discarded_records += 1;
discarded_bytes += payload.len() as u64;
partial = true;
tracing::warn!(error = %e, table = table, "replayer: deserialise failed — skipping record");
continue;
}
};
Expand All @@ -813,17 +854,73 @@ impl ChWriter {
table = table,
"replayer: CH insert failed — backing off"
);
self.metrics.inc_counter(
"rush_ingest_replay_records_total",
&[("backend", backend), ("outcome", "insert_error")],
1,
);
all_ok = false;
break;
}
self.metrics.inc_counter(
"rush_ingest_replay_records_total",
&[("backend", backend), ("outcome", "inserted")],
1,
);
}

self.metrics.observe_histogram(
"rush_ingest_replay_duration_ms",
&[("backend", backend)],
started.elapsed().as_secs_f64() * 1000.0,
);
if all_ok {
let n = drain.records.len();
self.buffer.commit(drain).await;
let committed = self.buffer.commit(drain).await;
if committed && discarded_records > 0 {
let labels = [
("backend", backend),
("reason", "payload_decode"),
("unit", "record"),
];
self.metrics.inc_counter(
"rush_ingest_spool_discarded_total",
&labels,
discarded_records,
);
self.metrics.inc_counter(
"rush_ingest_spool_discarded_bytes_total",
&labels,
discarded_bytes,
);
}
let outcome = if !committed {
"commit_error"
} else if partial {
"partial"
} else {
"committed"
};
self.metrics.inc_counter(
"rush_ingest_replay_batches_total",
&[("backend", backend), ("outcome", outcome)],
1,
);
if committed && !partial {
self.metrics.set_gauge(
"rush_ingest_replay_last_success_timestamp_seconds",
&[("backend", backend)],
chrono::Utc::now().timestamp() as f64,
);
}
tracing::info!(records = n, "replayer: batch replayed and committed");
backoff = Duration::from_secs(5); // reset on success
} else {
self.metrics.inc_counter(
"rush_ingest_replay_batches_total",
&[("backend", backend), ("outcome", "insert_error")],
1,
);
// CH is still down — back off and stop this replay pass.
// (drain not committed → retried next pass.) During graceful
// shutdown this intentionally waits rather than exiting;
Expand Down Expand Up @@ -1150,6 +1247,47 @@ mod batch_tests {
));
}

#[tokio::test]
async fn invalid_replay_payload_is_partial_not_successful_delivery() {
let test = TestWriter::new("http://127.0.0.1:9", BatchConfig::default());
// TestWriter's spool has a zero-byte cap; use a separate bounded spool.
let dir = test.1.join("replay");
let buffer = Arc::new(IngestBuffer::Disk(
crate::spool::Spool::open(&dir, 1024).unwrap(),
));
buffer
.append("untrusted-table-name", b"bad".to_vec())
.await
.unwrap();
let metrics = Arc::new(SelfMetrics::new());
let writer = ChWriter::new(
Client::default().with_url("http://127.0.0.1:9"),
buffer.clone(),
)
.with_metrics(metrics.clone());
let shutdown = ShutdownController::new();
shutdown.begin_drain();
let task = writer.spawn_replayer(shutdown.clone());
tokio::time::timeout(Duration::from_secs(2), async {
while buffer.segment_count() > 0 {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
shutdown.finish();
task.await.unwrap();
let text = metrics.render_prometheus();
assert!(text.contains("rush_ingest_spool_discarded_total{backend=\"disk\",reason=\"payload_decode\",unit=\"record\"} 1"));
assert!(
text.contains(
"rush_ingest_replay_batches_total{backend=\"disk\",outcome=\"partial\"} 1"
)
);
assert!(!text.contains("last_success_timestamp"));
assert!(!text.contains("untrusted-table-name"));
}

#[tokio::test]
async fn rejected_retries_do_not_enter_the_buffer() {
let test = TestWriter::new(
Expand Down
1 change: 1 addition & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ pub mod spool;
pub mod stats_engine;
pub mod usage_accumulator;
pub mod usage_tracker;
pub mod worker_metrics;

use clickhouse::Client;
use clickhouse::query::Query;
Expand Down
3 changes: 3 additions & 0 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3152,6 +3152,9 @@ async fn main() -> anyhow::Result<()> {
// Drain-worker-only: this process exists solely to drain the ingest buffer
// into ClickHouse. Don't serve HTTP, run engines, or load the firewall.
if drain_only {
let _metrics_server =
rush_api::worker_metrics::spawn(self_metrics.clone(), shutdown_controller.clone())
.await?;
tracing::info!(
backend = effective_backend,
"RUSH_DRAIN_WORKER_ONLY — draining ingest buffer to ClickHouse; not serving HTTP"
Expand Down
53 changes: 51 additions & 2 deletions src/object_store_spool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@
//! Object body = `{table}\n{payload}` (table names have no newline; the payload
//! is the same serde_json batch bytes the disk spool stores).

use crate::self_metrics::SelfMetrics;
use crate::spool::{record_spool_discard, record_spool_error};
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};

use bytes::Bytes;
Expand Down Expand Up @@ -67,6 +69,7 @@ impl Drop for AppendReservation<'_> {
}

pub struct ObjectStoreSpool {
metrics: OnceLock<Arc<SelfMetrics>>,
store: Arc<dyn ObjectStore>,
prefix: String,
max_bytes: u64,
Expand All @@ -82,6 +85,15 @@ pub struct ObjectStoreSpool {
}

impl ObjectStoreSpool {
pub fn attach_metrics(&self, metrics: Arc<SelfMetrics>) {
let _ = self.metrics.set(metrics);
}
fn record_error(&self, operation: &'static str) {
if let Some(metrics) = self.metrics.get() {
record_spool_error(metrics, "object_store", operation);
}
}

/// Build an S3/MinIO-backed buffer and seed counters from any objects left
/// over from a previous run (durable across restarts/pods).
pub async fn open_s3(
Expand Down Expand Up @@ -123,6 +135,7 @@ impl ObjectStoreSpool {
format!("{prefix}/")
};
let s = ObjectStoreSpool {
metrics: OnceLock::new(),
store,
prefix,
max_bytes,
Expand Down Expand Up @@ -296,6 +309,7 @@ impl ObjectStoreSpool {
.first_key_value()
.map(|(key, size)| (key.clone(), *size)),
Err(e) => {
self.record_error("list");
tracing::warn!(error = %e, "object-store buffer: list failed");
return None;
}
Expand All @@ -306,6 +320,7 @@ impl ObjectStoreSpool {
Ok(r) => match r.bytes().await {
Ok(b) => b,
Err(e) => {
self.record_error("read");
tracing::warn!(error = %e, key = %key, "buffer: get bytes failed");
return None;
}
Expand All @@ -317,6 +332,7 @@ impl ObjectStoreSpool {
continue;
}
Err(e) => {
self.record_error("get");
tracing::warn!(error = %e, key = %key, "buffer: get failed");
return None;
}
Expand All @@ -328,23 +344,36 @@ impl ObjectStoreSpool {
if !self.delete_batch(&key, size).await {
return None;
}
if let Some(metrics) = self.metrics.get() {
record_spool_discard(
metrics,
"object_store",
"frame_decode",
"object",
size,
);
}
continue; // try the next object
}
}
}
}

/// Delete a successfully-drained object.
pub async fn commit(&self, key: &OsPath, size: u64) {
pub async fn commit(&self, key: &OsPath, size: u64) -> bool {
if self.delete_batch(key, size).await {
self.committed.fetch_add(1, Ordering::Relaxed);
true
} else {
false
}
}

async fn delete_batch(&self, key: &OsPath, size: u64) -> bool {
let _transaction = self.transactions.read().await;
let mut index = self.index.lock().await;
if let Err(e) = self.store.delete(key).await {
self.record_error("delete");
tracing::warn!(error = %e, key = %key, "object-store buffer: delete failed");
return false;
}
Expand Down Expand Up @@ -423,6 +452,26 @@ mod tests {
}
}

#[tokio::test]
async fn corrupt_objects_are_discarded_without_inflating_committed_batches() {
let store = store();
store
.put(
&OsPath::from("ingest/0000000000000-00000000.batch"),
Bytes::from_static(b"broken").into(),
)
.await
.unwrap();
let spool = ObjectStoreSpool::open(store, "ingest", 1024).await.unwrap();
let metrics = Arc::new(SelfMetrics::new());
spool.attach_metrics(metrics.clone());
assert!(spool.next_batch().await.is_none());
assert!(spool.next_batch().await.is_none());
assert_eq!(spool.committed_total(), 0);
assert_eq!(spool.total_bytes(), 0);
assert!(metrics.render_prometheus().contains("rush_ingest_spool_discarded_total{backend=\"object_store\",reason=\"frame_decode\",unit=\"object\"} 1"));
}

#[tokio::test]
async fn drain_batches_keep_a_bounded_oldest_index_and_reuse_age_lookups() {
let st = store();
Expand Down
5 changes: 5 additions & 0 deletions src/shutdown.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ impl ShutdownController {
.await;
}

pub async fn wait_for_finish(&self) {
self.wait_until(|phase| phase == ShutdownPhase::Finished)
.await;
}

pub async fn wait_for_drain(&self) {
self.wait_until(|phase| phase >= ShutdownPhase::Draining)
.await;
Expand Down
Loading
Loading