Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions src/handlers/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -876,6 +876,36 @@ enum StreamKind {
Json,
}

const EXPORT_ROWS_BUCKETS: &[f64] = &[
0.0,
1.0,
10.0,
100.0,
1_000.0,
10_000.0,
50_000.0,
100_000.0,
500_000.0,
1_000_000.0,
];

struct SerializedRowsObservation {
metrics: Arc<crate::self_metrics::SelfMetrics>,
format: &'static str,
rows: u64,
outcome: &'static str,
}
impl Drop for SerializedRowsObservation {
fn drop(&mut self) {
self.metrics.observe_histogram_with(
"rush_export_serialized_rows",
&[("format", self.format), ("outcome", self.outcome)],
self.rows as f64,
EXPORT_ROWS_BUCKETS,
);
}
}

/// State for both CSV and JSON streaming. Only a single decoded row and its
/// serialized chunk are live at a time, so peak API memory is independent of
/// the requested row count.
Expand All @@ -888,6 +918,7 @@ struct ExportStreamState<T> {
max_bytes: u64,
progress: ExportProgress,
on_progress: Option<ProgressCallback>,
metrics: SerializedRowsObservation,
done: bool,
}

Expand All @@ -903,6 +934,11 @@ impl<T> ExportStreamState<T> {
return None;
}
self.done = true;
self.metrics.outcome = if self.progress.truncated {
"truncated"
} else {
"complete"
};
let suffix = match self.kind {
StreamKind::Csv if self.progress.truncated => {
b"# truncated: export byte limit reached\n".to_vec()
Expand All @@ -928,6 +964,7 @@ impl<T> ExportStreamState<T> {
}
}

#[allow(clippy::too_many_arguments)]
fn stream_response<T>(
cursor: RowCursor<T>,
prelude: Vec<u8>,
Expand All @@ -937,6 +974,7 @@ fn stream_response<T>(
max_rows: u64,
max_bytes: u64,
on_progress: Option<ProgressCallback>,
metrics: Arc<crate::self_metrics::SelfMetrics>,
) -> Response
where
T: clickhouse::RowOwned + clickhouse::RowRead + Send + 'static,
Expand All @@ -950,6 +988,15 @@ where
max_bytes,
progress: ExportProgress::default(),
on_progress,
metrics: SerializedRowsObservation {
metrics,
format: match kind {
StreamKind::Csv => "csv",
StreamKind::Json => "json",
},
rows: 0,
outcome: "cancelled",
},
done: false,
};

Expand All @@ -960,6 +1007,7 @@ where
if let Some(prelude) = state.prelude.take() {
if prelude.len() as u64 + STREAM_SUFFIX_RESERVE > state.max_bytes {
state.done = true;
state.metrics.outcome = "metadata_limit";
return Some((
Err::<Bytes, std::io::Error>(std::io::Error::other(
"export metadata exceeds the byte limit",
Expand All @@ -981,6 +1029,7 @@ where
Err(_error) => {
tracing::error!(reason = "row_serialization", "export stream failed");
state.done = true;
state.metrics.outcome = "serialization_error";
return Some((Err(std::io::Error::other("export stream failed")), state));
}
};
Expand All @@ -998,6 +1047,7 @@ where
return state.finish_chunk().map(|suffix| (Ok(suffix), state));
}
state.progress.rows += 1;
state.metrics.rows = state.progress.rows;
state.progress.bytes += chunk.len() as u64;
state.notify();
Some((Ok(Bytes::from(chunk)), state))
Expand All @@ -1006,6 +1056,7 @@ where
Err(_error) => {
tracing::error!(reason = "cursor_read", "export stream failed");
state.done = true;
state.metrics.outcome = "cursor_error";
Some((Err(std::io::Error::other("export stream failed")), state))
}
}
Expand Down Expand Up @@ -1043,6 +1094,7 @@ where
/// Errors mid-stream terminate the body with an `io::Error`; axum surfaces that as a
/// truncated/aborted response. The initial query has already executed by the time the
/// first row is pulled, so query-level failures still abort the download cleanly.
#[allow(clippy::too_many_arguments)]
pub fn stream_csv_response<T>(
cursor: RowCursor<T>,
prelude: String,
Expand All @@ -1051,6 +1103,7 @@ pub fn stream_csv_response<T>(
max_rows: u64,
max_bytes: u64,
on_progress: Option<ProgressCallback>,
metrics: Arc<crate::self_metrics::SelfMetrics>,
) -> Response
where
T: clickhouse::RowOwned + clickhouse::RowRead + Send + 'static,
Expand All @@ -1064,6 +1117,7 @@ where
max_rows,
max_bytes,
on_progress,
metrics,
)
}

Expand All @@ -1083,13 +1137,15 @@ pub fn json_query_preamble(query: serde_json::Value) -> Vec<u8> {
bytes
}

#[allow(clippy::too_many_arguments)]
pub fn stream_json_response<T>(
cursor: RowCursor<T>,
prelude: Vec<u8>,
filename: &str,
max_rows: u64,
max_bytes: u64,
on_progress: Option<ProgressCallback>,
metrics: Arc<crate::self_metrics::SelfMetrics>,
) -> Response
where
T: clickhouse::RowOwned + clickhouse::RowRead + serde::Serialize + Send + 'static,
Expand All @@ -1103,13 +1159,53 @@ where
max_rows,
max_bytes,
on_progress,
metrics,
)
}

#[cfg(test)]
mod tests {
use super::*;

#[tokio::test]
async fn serialized_row_metrics_follow_actual_cursor_output() {
#[derive(clickhouse::Row, serde::Deserialize, serde::Serialize)]
struct TestRow {
value: u64,
}
let mock = clickhouse::test::Mock::new();
mock.add(clickhouse::test::handlers::provide([
TestRow { value: 1 },
TestRow { value: 2 },
]));
let ch = clickhouse::Client::default()
.with_url(mock.url())
.with_validation(false);
let cursor = ch.query("SELECT value").fetch::<TestRow>().unwrap();
let metrics = Arc::new(crate::self_metrics::SelfMetrics::new());
let response = stream_json_response(
cursor,
json_query_preamble(serde_json::json!({})),
"test.json",
10,
1024 * 1024,
None,
metrics.clone(),
);
let body = axum::body::to_bytes(response.into_body(), 1024 * 1024)
.await
.unwrap();
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&body).unwrap()["count"],
2
);
assert!(
metrics.render_prometheus().contains(
"rush_export_serialized_rows_sum{format=\"json\",outcome=\"complete\"} 2"
)
);
}

#[test]
fn async_threshold_applies_to_every_large_effective_request() {
assert!(!requires_async(0, sync_max_rows()));
Expand Down
15 changes: 10 additions & 5 deletions src/handlers/logs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1025,6 +1025,8 @@ pub async fn export_logs(
}
};
let budget = guard.budget().clone();
let deadline = tokio::time::Instant::now()
+ std::time::Duration::from_secs(budget.request_timeout_secs);
let progress = background_state
.export_jobs
.progress_callback(&background_job_id);
Expand All @@ -1049,6 +1051,7 @@ pub async fn export_logs(
limit,
max_bytes,
Some(progress),
background_state.self_metrics.clone(),
),
export::ExportFormat::Json => export::stream_json_response(
cursor,
Expand All @@ -1057,22 +1060,22 @@ pub async fn export_logs(
limit,
max_bytes,
Some(progress),
background_state.self_metrics.clone(),
),
};
let response = crate::query_governor::retain_artifact_until_body_deadline(
response, guard, deadline,
);
background_state
.export_jobs
.write_response(&background_job_id, response)
.await
};
let result = crate::query_governor::with_budget(
budget.clone(),
tokio::time::timeout(
std::time::Duration::from_secs(budget.request_timeout_secs),
work,
),
tokio::time::timeout_at(deadline, work),
)
.await;
drop(guard);
match result {
Ok(Ok(())) => {}
_ if background_state
Expand Down Expand Up @@ -1138,6 +1141,7 @@ pub async fn export_logs(
limit,
max_bytes,
None,
state.self_metrics.clone(),
))
}
export::ExportFormat::Json => {
Expand Down Expand Up @@ -1168,6 +1172,7 @@ pub async fn export_logs(
limit,
max_bytes,
None,
state.self_metrics.clone(),
))
}
}
Expand Down
15 changes: 10 additions & 5 deletions src/handlers/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,8 @@ pub async fn export_query(
}
};
let budget = guard.budget().clone();
let deadline = tokio::time::Instant::now()
+ std::time::Duration::from_secs(budget.request_timeout_secs);
let progress = background_state
.export_jobs
.progress_callback(&background_job_id);
Expand Down Expand Up @@ -453,6 +455,7 @@ pub async fn export_query(
limit,
max_bytes,
Some(progress),
background_state.self_metrics.clone(),
),
export::ExportFormat::Json => export::stream_json_response(
cursor,
Expand All @@ -461,22 +464,22 @@ pub async fn export_query(
limit,
max_bytes,
Some(progress),
background_state.self_metrics.clone(),
),
};
let response = crate::query_governor::retain_artifact_until_body_deadline(
response, guard, deadline,
);
background_state
.export_jobs
.write_response(&background_job_id, response)
.await
};
let result = crate::query_governor::with_budget(
budget.clone(),
tokio::time::timeout(
std::time::Duration::from_secs(budget.request_timeout_secs),
work,
),
tokio::time::timeout_at(deadline, work),
)
.await;
drop(guard);
match result {
Ok(Ok(())) => {}
_ if background_state
Expand Down Expand Up @@ -547,6 +550,7 @@ pub async fn export_query(
limit,
max_bytes,
None,
state.self_metrics.clone(),
))
}
export::ExportFormat::Json => {
Expand Down Expand Up @@ -577,6 +581,7 @@ pub async fn export_query(
limit,
max_bytes,
None,
state.self_metrics.clone(),
))
}
}
Expand Down
Loading
Loading