From 69f69f6a6cc6e565b03caffeb99a07965b826dbc Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:45:47 +0000 Subject: [PATCH 1/3] feat(TC-2917): DuckDB disk headroom, spill limit and startup cache sweep Writes reserve their local file size against the node-wide disk budget and are refused (507 duckdb_disk_headroom) below storage.duckdb.min_free_bytes; a running write is interrupted if free space drops below the floor; a statement spilling past storage.duckdb.max_temp_directory_size is the same 507. Defaults (2 GiB floor, 1 GiB spill) apply only with chunked artifacts enabled. At startup the node removes DuckDB spill directories and staging or renamed-aside cache names left by a stopped process. --- CHANGELOG.md | 2 + docs/operations/chunked-sql-artifacts.md | 34 +- tinycloud-core/src/duckdb/database.rs | 67 +++- tinycloud-core/src/duckdb/disk.rs | 186 +++++++++++ tinycloud-core/src/duckdb/mod.rs | 1 + tinycloud-core/src/duckdb/service.rs | 401 ++++++++++++++++++++++- tinycloud-core/src/duckdb/storage.rs | 46 ++- tinycloud-core/src/duckdb/types.rs | 2 + tinycloud-node-server/src/config.rs | 71 ++++ tinycloud-node-server/src/lib.rs | 14 +- tinycloud-node-server/src/prometheus.rs | 7 +- tinycloud-node-server/src/routes/mod.rs | 16 + tinycloud.toml | 5 + 13 files changed, 820 insertions(+), 32 deletions(-) create mode 100644 tinycloud-core/src/duckdb/disk.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index cc10bfdca..ea20a28ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - Policy/v3: minting a session and registering a policy no longer re-verify signatures or re-run the claim, presentation, credential admission and allow evaluations while holding their spaces' write order and row locks. Under the locks they check that the challenge, registration and roots are unchanged; those decisions were made before the locks, at the same time, from the same inputs. Refusal codes are unchanged, except that `credential-evaluation-changed` and `policy-evaluation-changed` can no longer occur. Details: TC-2986. - Artifact-part writers prioritize their own predecessor reclamation, keep replacement contention retryable, and distinguish capacity refusals from space-growth limits; existing database layouts can checkpoint before adopting WAL parts. Details: TC-2917. - DuckDB cache cleanup no longer holds up lookups of other databases while files are deleted; a spill directory left by a stopped process no longer blocks hydration or discard, and a failed import leaves no staged or imported file behind. Details: TC-2917. +- DuckDB keeps disk headroom: a write or import that would leave less than `storage.duckdb.min_free_bytes` free, a write still running when free space drops below it, and a statement that spills past `storage.duckdb.max_temp_directory_size` answer `507 duckdb_disk_headroom` and are never applied or acknowledged; each is counted as `tinycloud_disk_headroom_refusals_total{engine="duckdb"}`. Reservations share the node-wide disk budget with SQL. Unset, the floor is 2 GiB (provisional until production headroom is measured) and the spill limit 1 GiB per connection when chunked artifact storage is enabled; otherwise both are off. Details: TC-2917. +- At startup the node removes DuckDB spill directories and staging or renamed-aside cache files left by a stopped process, before it serves; the cache files themselves stay. Details: TC-2917. - DuckDB saves a complete checkpoint when a write changes its main file, and uses incremental WAL persistence only when that file still matches the stored checkpoint, so acknowledged writes remain recoverable after restart. Details: TC-2990. - Opt-in SQL artifact parts also stream cumulative WAL updates, retaining the checkpoint until a full checkpoint is needed. Details: TC-2917. - DuckDB can persist checkpoints, cumulative WAL and imports through the same opt-in bounded artifact parts. Details: TC-2917. diff --git a/docs/operations/chunked-sql-artifacts.md b/docs/operations/chunked-sql-artifacts.md index e2f91eb78..5fc0a1ea9 100644 --- a/docs/operations/chunked-sql-artifacts.md +++ b/docs/operations/chunked-sql-artifacts.md @@ -139,10 +139,31 @@ For disk headroom, each successfully persisted open database retains at most the configured artifact ceiling in checkpoint plus cumulative WAL. A full checkpoint also needs an owned temporary copy while streaming, and old/new versions overlap in PostgreSQL until their deletions commit. PostgreSQL can reuse deleted storage after vacuum; relation files need not -shrink. Size refusal occurs after native execution; -the separate snapshot/headroom acceptance must cover transient oversized -writes and temporary copies before enabling. Operators check CVM disk headroom -against the number and ceilings of open databases, plus that transient cost. +shrink. Size refusal occurs after native execution. + +DuckDB keeps disk headroom on the filesystem behind its cache. With parts +enabled, `storage.duckdb.min_free_bytes` defaults to 2 GiB and +`storage.duckdb.max_temp_directory_size` to 1 GiB per connection; with parts +off both stay off unless set. The 2 GiB floor is provisional until the +`storage_headroom` event shows real headroom on production nodes; operators tune +it by config. + +- Before a write runs, it reserves its local database files' size (the bound + of the snapshot copy its save can take); an import reserves twice its size. + Reservations share the node-wide budget with SQL. A reservation that would + leave less than the floor is refused with `507 duckdb_disk_headroom` and + nothing runs. +- While a write runs, it is interrupted if free space falls below what other + operations hold plus the floor. The write is not applied, not saved and not + acknowledged; it answers `507 duckdb_disk_headroom`. +- A statement that spills past the per-connection limit also answers + `507 duckdb_disk_headroom`. Queries and exports reserve nothing and are not + interrupted for headroom; only the spill limit applies to them. +- Hydration is not gated: it writes at most the artifact ceiling and is freed + at eviction. +- At startup, before it serves, the node removes what a stopped process left in + the DuckDB cache: spill directories and staging or renamed-aside files. The + cache files themselves stay, because a legacy database reopens them. Artifact, account and single-statement capacity refusals return 507. Existing per-space growth refusals retain 402. A concurrent replacement or contended @@ -157,6 +178,5 @@ the local main file's length and CID with the stored checkpoint under the actor fence. A mismatch requires a complete checkpoint, including blocks written directly to the main file by a large insert. -The feature remains off. DuckDB disk and spill admission still needs its -separate acceptance before activation, alongside conversion and the -ceiling-load benchmark. +The feature remains off. Conversion and the ceiling-load benchmark still need +their separate acceptance before activation. diff --git a/tinycloud-core/src/duckdb/database.rs b/tinycloud-core/src/duckdb/database.rs index b121c0e8e..00fd54a88 100644 --- a/tinycloud-core/src/duckdb/database.rs +++ b/tinycloud-core/src/duckdb/database.rs @@ -18,7 +18,9 @@ use crate::request_telemetry::{self, QueryTimer, RequestContext}; use super::{ caveats::DuckDbCaveats, - describe, parser, + describe, + disk::{self, DuckDbDiskPolicy}, + parser, storage::{self, StorageMode}, types::*, }; @@ -241,6 +243,34 @@ pub(crate) fn spawn_actor_with_parts( idle_timeout_secs: u64, max_memory_per_connection: String, databases: Arc>, +) -> DatabaseHandle { + spawn_actor_with_disk( + space_id, + db_name, + base_path, + memory_threshold, + force_file, + idle_timeout_secs, + max_memory_per_connection, + databases, + DuckDbDiskPolicy::default(), + ) +} + +/// [`spawn_actor_with_parts`] under `disk`: its connections spill at most +/// `disk.max_temp_directory_bytes`, and each statement runs under the disk +/// watcher. +#[allow(clippy::too_many_arguments)] +pub(crate) fn spawn_actor_with_disk( + space_id: String, + db_name: String, + base_path: String, + memory_threshold: u64, + force_file: bool, + idle_timeout_secs: u64, + max_memory_per_connection: String, + databases: Arc>, + disk: DuckDbDiskPolicy, ) -> DatabaseHandle { let (tx, mut rx) = mpsc::channel::<(Option>, DbMessage)>(32); let id = NEXT_ACTOR_ID.fetch_add(1, Ordering::Relaxed); @@ -265,7 +295,11 @@ pub(crate) fn spawn_actor_with_parts( }; let mut conn = { let _context = request_telemetry::enter(initial_context); - match storage::open_connection(opening_path, &max_memory_per_connection) { + match storage::open_spill_capped_connection( + opening_path, + &max_memory_per_connection, + disk.max_temp_directory_bytes, + ) { Ok(c) => c, Err(e) => { tracing::error!("Failed to open database"); @@ -335,26 +369,31 @@ pub(crate) fn spawn_actor_with_parts( without_growth, response_tx, } => { - let result = if without_growth { - handle_message_without_growth( - &conn, - &request, - &caveats, - &ability, - arrow_format, - ) - } else { - handle_message(&conn, &request, &caveats, &ability, arrow_format, None) - }; + let budget_key = disk::budget_key(&space_id, &db_name); + let write = disk::writes(&request); + let result = disk.watch(&budget_key, &file_path, &conn, write, || { + if without_growth { + handle_message_without_growth( + &conn, + &request, + &caveats, + &ability, + arrow_format, + ) + } else { + handle_message(&conn, &request, &caveats, &ability, arrow_format, None) + } + }); // Post-write promotion check if result.is_ok() && matches!(mode, StorageMode::InMemory) { if let Ok(size) = storage::database_size(&conn) { if size > memory_threshold { - match storage::promote_to_file( + match storage::promote_to_spill_capped_file( &conn, &file_path, &max_memory_per_connection, + disk.max_temp_directory_bytes, ) { Ok(new_conn) => { conn = new_conn; diff --git a/tinycloud-core/src/duckdb/disk.rs b/tinycloud-core/src/duckdb/disk.rs new file mode 100644 index 000000000..ecffc7b75 --- /dev/null +++ b/tinycloud-core/src/duckdb/disk.rs @@ -0,0 +1,186 @@ +//! Disk headroom for DuckDB databases (TC-2917). +//! +//! The node keeps `min_free_bytes` free on the filesystem behind the DuckDB +//! cache. Before a write or an import runs, the bytes it can need are reserved +//! in the node-wide [`DiskBudget`] that SQL uses too, so two databases cannot +//! count the same free bytes. While a statement runs, a watcher interrupts it +//! once free space falls below what other operations hold plus the floor. A +//! refused or interrupted write answers [`DuckDbError::DiskHeadroom`] and is +//! never acknowledged. + +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::time::Duration; + +use crate::sql::durability::{ + DiskBudget, FreeSpace, ReservationGuard, ReservationRefused, StatvfsFreeSpace, +}; + +use super::types::{DuckDbError, DuckDbRequest}; + +/// How often the watcher reads free space while a statement runs. +const WATCH_INTERVAL: Duration = Duration::from_millis(50); + +/// The DuckDB disk settings. The default is off: no reservations, no +/// free-space reads and DuckDB's own spill limit. +#[derive(Clone)] +pub struct DuckDbDiskPolicy { + /// Free bytes that must remain after every in-flight operation's + /// reservation. 0 turns the headroom check and the watcher off. + pub min_free_bytes: u64, + /// The most bytes one connection may spill to its temporary directory + /// (DuckDB `max_temp_directory_size`). 0 keeps DuckDB's default. + pub max_temp_directory_bytes: u64, + pub budget: Arc, + pub free_space: Arc, +} + +impl Default for DuckDbDiskPolicy { + fn default() -> Self { + Self { + min_free_bytes: 0, + max_temp_directory_bytes: 0, + budget: Arc::new(DiskBudget::default()), + free_space: Arc::new(StatvfsFreeSpace), + } + } +} + +impl std::fmt::Debug for DuckDbDiskPolicy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DuckDbDiskPolicy") + .field("min_free_bytes", &self.min_free_bytes) + .field("max_temp_directory_bytes", &self.max_temp_directory_bytes) + .finish_non_exhaustive() + } +} + +/// The budget key of one DuckDB database. SQL keys never start with this. +pub(crate) fn budget_key(space: &str, db_name: &str) -> String { + format!("duckdb:{space}/{db_name}") +} + +/// Requests that can change the database and are saved afterwards: they +/// reserve room and run under the watcher. Queries, exports and `describe` +/// do neither; only the spill limit applies to them. +pub(crate) fn writes(request: &DuckDbRequest) -> bool { + !matches!( + request, + DuckDbRequest::Query { .. } + | DuckDbRequest::Describe + | DuckDbRequest::Export + | DuckDbRequest::ExportToKv { .. } + ) +} + +impl DuckDbDiskPolicy { + pub(crate) fn headroom_checked(&self) -> bool { + self.min_free_bytes > 0 + } + + /// Reserve `bytes` for one operation of `key`, measured against the + /// filesystem behind `dir`. `None` when the check is off. + pub(crate) fn reserve( + &self, + key: &str, + bytes: u64, + dir: &Path, + ) -> Result, DuckDbError> { + if !self.headroom_checked() { + return Ok(None); + } + match self.budget.reserve( + key, + bytes, + 0, + self.free_space.as_ref(), + existing_ancestor(dir), + self.min_free_bytes, + ) { + Ok(()) => Ok(Some(ReservationGuard::new( + self.budget.clone(), + key.to_string(), + ))), + Err(ReservationRefused::Headroom) => Err(DuckDbError::DiskHeadroom), + // Free space that cannot be read is refused, never allowed. + Err(ReservationRefused::Unknown) => Err(DuckDbError::Internal( + "free disk space could not be read".into(), + )), + } + } + + /// Run `statement` on `conn`. A write (`write`) is interrupted if free + /// space behind `dir` falls below what other operations hold plus the + /// floor. A statement stopped that way, or one that hit the spill limit, + /// is [`DuckDbError::DiskHeadroom`]. + pub(crate) fn watch( + &self, + key: &str, + dir: &Path, + conn: &duckdb::Connection, + write: bool, + statement: impl FnOnce() -> Result, + ) -> Result { + if !write || !self.headroom_checked() { + return statement().map_err(|error| self.classify(error)); + } + let interrupt = conn.interrupt_handle(); + let tripped = AtomicBool::new(false); + let dir = existing_ancestor(dir); + let result = std::thread::scope(|scope| { + let (stop, stopped) = mpsc::channel::<()>(); + let (tripped, dir) = (&tripped, &dir); + scope.spawn(move || { + while let Err(mpsc::RecvTimeoutError::Timeout) = + stopped.recv_timeout(WATCH_INTERVAL) + { + let floor = self + .budget + .reserved_excluding(key) + .saturating_add(self.min_free_bytes); + // Unreadable free space stops the statement, like the gate. + let short = self + .free_space + .free_bytes(dir) + .map_or(true, |free| free < floor); + if short { + tripped.store(true, Ordering::Relaxed); + interrupt.interrupt(); + break; + } + } + }); + let result = statement(); + drop(stop); + result + }); + match result { + Err(_) if tripped.load(Ordering::Relaxed) => Err(DuckDbError::DiskHeadroom), + other => other.map_err(|error| self.classify(error)), + } + } + + /// A statement that ran past the spill limit is the disk, not the caller. + fn classify(&self, error: DuckDbError) -> DuckDbError { + let spill_limit = self.max_temp_directory_bytes > 0 + && matches!( + &error, + DuckDbError::DuckDb(message) | DuckDbError::Internal(message) + if message.contains("max_temp_directory_size") + ); + if spill_limit { + DuckDbError::DiskHeadroom + } else { + error + } + } +} + +/// The nearest existing ancestor of `path`: where `statvfs` can be asked how +/// much room the filesystem has before the file or its directory exists. +fn existing_ancestor(path: &Path) -> &Path { + path.ancestors() + .find(|ancestor| ancestor.exists()) + .unwrap_or(path) +} diff --git a/tinycloud-core/src/duckdb/mod.rs b/tinycloud-core/src/duckdb/mod.rs index 5bcde887a..a40c39f81 100644 --- a/tinycloud-core/src/duckdb/mod.rs +++ b/tinycloud-core/src/duckdb/mod.rs @@ -1,6 +1,7 @@ pub mod caveats; pub mod database; pub mod describe; +pub mod disk; pub mod parser; pub mod service; pub mod storage; diff --git a/tinycloud-core/src/duckdb/service.rs b/tinycloud-core/src/duckdb/service.rs index 1bce269aa..b9ba82c75 100644 --- a/tinycloud-core/src/duckdb/service.rs +++ b/tinycloud-core/src/duckdb/service.rs @@ -16,7 +16,8 @@ use crate::request_telemetry::{self, QueryTimer}; use super::{ caveats::DuckDbCaveats, - database::{cache_file_paths, remove_cache_entry, spawn_actor_with_parts, DatabaseHandle}, + database::{cache_file_paths, remove_cache_entry, spawn_actor_with_disk, DatabaseHandle}, + disk::{self, DuckDbDiskPolicy}, storage, types::*, }; @@ -45,6 +46,7 @@ pub struct DuckDbService { idle_timeout_secs: u64, max_memory_per_connection: String, artifact_repository: Arc, + disk: DuckDbDiskPolicy, } struct ExecuteOptions { @@ -84,9 +86,49 @@ impl DuckDbService { idle_timeout_secs, max_memory_per_connection, artifact_repository, + disk: DuckDbDiskPolicy::default(), } } + /// Keep disk headroom and cap spilling under `disk` (off by default). + pub fn with_disk_policy(mut self, disk: DuckDbDiskPolicy) -> Self { + self.disk = disk; + self + } + + /// Remove what a stopped process left in the cache: DuckDB spill + /// directories and staging or renamed-aside names. Run once at startup, + /// before any actor exists. The cache files themselves are kept, because + /// a legacy database reopens them. Returns how many entries were removed. + pub async fn sweep_stale_cache_files(&self) -> usize { + let base = PathBuf::from(&self.base_path); + tokio::task::spawn_blocking(move || { + let mut removed = 0; + let Ok(spaces) = std::fs::read_dir(&base) else { + return 0; + }; + for space in spaces.filter_map(Result::ok) { + let Ok(entries) = std::fs::read_dir(space.path()) else { + continue; + }; + for entry in entries.filter_map(Result::ok) { + let name = entry.file_name(); + if is_stale_cache_name(&name.to_string_lossy()) + && remove_cache_entry(&entry.path()).is_ok() + { + removed += 1; + } + } + } + if removed > 0 { + tracing::info!(removed, "Removed DuckDB cache leftovers at startup"); + } + removed + }) + .await + .unwrap_or(0) + } + pub async fn execute( &self, space: &SpaceId, @@ -160,6 +202,17 @@ impl DuckDbService { let _operation = operation_lock.lock().await; let handle = self.handle(space, db_name).await?; + // Room for this write and for the snapshot copy its save can take, + // which is at most the local database files. + let _reservation = if disk::writes(&request) { + let cache_path = self.cache_path(space, db_name); + let local_bytes = local_file_bytes(&cache_path).await; + self.disk + .reserve(&disk::budget_key(&key.0, &key.1), local_bytes, &cache_path)? + } else { + None + }; + let result = handle .execute(request, caveats, ability, arrow_format, without_growth) .await; @@ -272,6 +325,13 @@ impl DuckDbService { ) .await; let _operation = operation_lock.lock().await; + // An import holds the staged file and, when saved in parts, a snapshot + // copy of it: twice its size on disk. + let _reservation = self.disk.reserve( + &disk::budget_key(&space.to_string(), db_name), + (data.len() as u64).saturating_mul(2), + &PathBuf::from(&self.base_path).join(space.to_string()), + )?; let dir = std::path::PathBuf::from(&self.base_path).join(space.to_string()); tokio::fs::create_dir_all(&dir) @@ -419,7 +479,7 @@ impl DuckDbService { .databases .entry(key) .or_insert_with(|| { - spawn_actor_with_parts( + spawn_actor_with_disk( space.to_string(), db_name.to_string(), self.base_path.clone(), @@ -432,6 +492,7 @@ impl DuckDbService { self.idle_timeout_secs, self.max_memory_per_connection.clone(), self.databases.clone(), + self.disk.clone(), ) }) .clone()) @@ -786,6 +847,40 @@ fn duckdb_wal_path(path: &Path) -> PathBuf { PathBuf::from(format!("{}.wal", path.display())) } +/// The bytes of the local main file and WAL at `path` (0 for absent files). +async fn local_file_bytes(path: &Path) -> u64 { + let mut total = 0u64; + for file in [path.to_path_buf(), duckdb_wal_path(path)] { + if let Ok(metadata) = tokio::fs::metadata(&file).await { + total = total.saturating_add(metadata.len()); + } + } + total +} + +/// A DuckDB spill directory (`.duckdb.tmp`) or a staging / renamed-aside +/// name (`.duckdb.tmp..`, `.duckdb.wal.tmp..`): never +/// live once the process that made it has stopped. +fn is_stale_cache_name(name: &str) -> bool { + if name.ends_with(".duckdb.tmp") { + return true; + } + let Some((_, suffix)) = name + .rsplit_once(".duckdb.tmp.") + .or_else(|| name.rsplit_once(".duckdb.wal.tmp.")) + else { + return false; + }; + matches!( + suffix.split_once('.'), + Some((pid, n)) + if !pid.is_empty() + && !n.is_empty() + && pid.bytes().all(|b| b.is_ascii_digit()) + && n.bytes().all(|b| b.is_ascii_digit()) + ) +} + /// `DuckDbError` has no retryable-unavailable variant (adding one would reach /// every route mapping), so an exhausted `ArtifactChanged` stays `Internal`. /// The artifact size limits answer 507, as for SQL (TC-2961). @@ -2527,4 +2622,306 @@ mod tests { assert!(dir_entries(&dir).is_empty(), "{:?}", dir_entries(&dir)); assert!(!service.databases.contains_key(&key)); } + + /// Free space the test sets; a node's real `statvfs` cannot be steered. + struct FakeFree(std::sync::atomic::AtomicU64); + + impl crate::sql::durability::FreeSpace for FakeFree { + fn free_bytes(&self, _: &Path) -> std::io::Result { + Ok(self.0.load(std::sync::atomic::Ordering::SeqCst)) + } + } + + fn disk_policy(free: &Arc, min_free_bytes: u64) -> DuckDbDiskPolicy { + DuckDbDiskPolicy { + min_free_bytes, + free_space: free.clone(), + ..Default::default() + } + } + + async fn count_rows(service: &DuckDbService, space: &SpaceId) -> Vec> { + query_rows(service, space, "SELECT count(*) FROM t") + .await + .unwrap() + } + + #[tokio::test] + async fn duckdb_write_short_of_headroom_is_refused_before_it_runs() { + let repo = artifact_repository().await; + let cache = TempDir::new().unwrap(); + let free = Arc::new(FakeFree((64u64 << 20).into())); + let service = + file_service(&cache, repo.clone()).with_disk_policy(disk_policy(&free, 8 << 20)); + let space = test_space_id("headroom-refused"); + service + .execute( + &space, + "main", + write_request( + "INSERT INTO t VALUES (1)", + Some("CREATE TABLE t(i INTEGER)"), + ), + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await + .unwrap(); + let saved = repo + .load_meta("duckdb", &space.to_string(), "main") + .await + .unwrap() + .unwrap() + .revision; + + free.0 + .store((8 << 20) - 1, std::sync::atomic::Ordering::SeqCst); + let refused = service + .execute( + &space, + "main", + write_request("INSERT INTO t VALUES (2)", None), + None, + "tinycloud.duckdb/write".into(), + false, + ) + .await; + assert!( + matches!(refused, Err(DuckDbError::DiskHeadroom)), + "{refused:?}" + ); + let imported = service + .import_db(&space, "main", &large_duckdb_file()) + .await; + assert!( + matches!(imported, Err(DuckDbError::DiskHeadroom)), + "{imported:?}" + ); + + // Reads still run while the disk is short; nothing was applied or saved. + assert_eq!( + count_rows(&service, &space).await, + vec![vec![DuckDbValue::Integer(1)]] + ); + let meta = repo + .load_meta("duckdb", &space.to_string(), "main") + .await + .unwrap() + .unwrap(); + assert_eq!(meta.revision, saved); + assert_eq!(service.disk.budget.reserved(), 0); + + free.0.store(64 << 20, std::sync::atomic::Ordering::SeqCst); + service + .execute( + &space, + "main", + write_request("INSERT INTO t VALUES (2)", None), + None, + "tinycloud.duckdb/write".into(), + false, + ) + .await + .unwrap(); + assert_eq!( + count_rows(&service, &space).await, + vec![vec![DuckDbValue::Integer(2)]] + ); + } + + #[tokio::test] + async fn duckdb_reservation_counts_other_databases_in_flight() { + let repo = artifact_repository().await; + let cache = TempDir::new().unwrap(); + let free = Arc::new(FakeFree((64u64 << 20).into())); + let policy = disk_policy(&free, 8 << 20); + let service = file_service(&cache, repo).with_disk_policy(policy.clone()); + let space = test_space_id("headroom-shared"); + // Another database (SQL or DuckDB) holds most of the free space. + policy + .budget + .reserve("sql:other", 60 << 20, 0, free.as_ref(), cache.path(), 0) + .unwrap(); + let refused = service + .execute( + &space, + "main", + write_request( + "INSERT INTO t VALUES (1)", + Some("CREATE TABLE t(i INTEGER)"), + ), + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await; + assert!( + matches!(refused, Err(DuckDbError::DiskHeadroom)), + "{refused:?}" + ); + policy.budget.release("sql:other"); + service + .execute( + &space, + "main", + write_request( + "INSERT INTO t VALUES (1)", + Some("CREATE TABLE t(i INTEGER)"), + ), + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn duckdb_statement_is_stopped_when_the_disk_runs_short_while_it_runs() { + let repo = artifact_repository().await; + let cache = TempDir::new().unwrap(); + let free = Arc::new(FakeFree((64u64 << 20).into())); + let service = + file_service(&cache, repo.clone()).with_disk_policy(disk_policy(&free, 8 << 20)); + let space = test_space_id("headroom-watch"); + service + .execute( + &space, + "main", + write_request("INSERT INTO t VALUES (1)", Some("CREATE TABLE t(i BIGINT)")), + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await + .unwrap(); + let saved = repo + .load_meta("duckdb", &space.to_string(), "main") + .await + .unwrap() + .unwrap() + .revision; + + // The disk runs short once the statement is under way. + let shrink = { + let free = free.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + free.0.store(0, std::sync::atomic::Ordering::SeqCst); + }) + }; + let stopped = service + .execute( + &space, + "main", + write_request( + "INSERT INTO t SELECT i FROM range(2000000000) r(i) WHERE md5(i::VARCHAR) <> ''", + None, + ), + None, + "tinycloud.duckdb/write".into(), + false, + ) + .await; + shrink.await.unwrap(); + assert!( + matches!(stopped, Err(DuckDbError::DiskHeadroom)), + "{stopped:?}" + ); + + free.0.store(64 << 20, std::sync::atomic::Ordering::SeqCst); + assert_eq!( + count_rows(&service, &space).await, + vec![vec![DuckDbValue::Integer(1)]] + ); + let meta = repo + .load_meta("duckdb", &space.to_string(), "main") + .await + .unwrap() + .unwrap(); + assert_eq!(meta.revision, saved); + } + + #[tokio::test] + async fn duckdb_statement_past_the_spill_limit_is_the_disk() { + let repo = artifact_repository().await; + let cache = TempDir::new().unwrap(); + let service = DuckDbService::new( + cache.path().to_string_lossy().into_owned(), + 0, + 300, + "16MB".into(), + repo, + ) + .with_disk_policy(DuckDbDiskPolicy { + max_temp_directory_bytes: 1 << 20, + ..Default::default() + }); + let space = test_space_id("spill-limit"); + // 2M rows under a 16 MB memory limit must spill, past the 1 MiB limit. + let big = service + .execute( + &space, + "main", + write_request( + "INSERT INTO t SELECT i, md5(i::VARCHAR) FROM range(2000000) r(i) ORDER BY 2", + Some("CREATE TABLE t(i BIGINT, s VARCHAR)"), + ), + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await; + assert!(matches!(big, Err(DuckDbError::DiskHeadroom)), "{big:?}"); + // The database stays usable for work that fits. + service + .execute( + &space, + "main", + write_request( + "INSERT INTO t SELECT i, md5(i::VARCHAR) FROM range(1000) r(i)", + Some("CREATE TABLE IF NOT EXISTS t(i BIGINT, s VARCHAR)"), + ), + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await + .unwrap(); + let rows = query_rows(&service, &space, "SELECT count(*) FROM t") + .await + .unwrap(); + assert_eq!(rows, vec![vec![DuckDbValue::Integer(1000)]]); + } + + #[tokio::test] + async fn duckdb_startup_sweep_removes_leftovers_and_keeps_cache_files() { + let cache = TempDir::new().unwrap(); + let service = service(&cache, artifact_repository().await); + let dir = cache.path().join("space-a"); + std::fs::create_dir_all(dir.join("main.duckdb.tmp")).unwrap(); + std::fs::write(dir.join("main.duckdb.tmp").join("spill"), b"x").unwrap(); + std::fs::create_dir_all(dir.join("main.duckdb.tmp.4242.7")).unwrap(); + for name in [ + "main.duckdb", + "main.duckdb.wal", + "main.duckdb.tmp.4242.8", + "main.duckdb.wal.tmp.4242.9", + "main.duckdb.tmp.notapid.1", + "notes.db", + ] { + std::fs::write(dir.join(name), b"x").unwrap(); + } + assert_eq!(service.sweep_stale_cache_files().await, 4); + assert_eq!( + dir_entries(&dir), + vec![ + "main.duckdb", + "main.duckdb.tmp.notapid.1", + "main.duckdb.wal", + "notes.db" + ] + ); + } } diff --git a/tinycloud-core/src/duckdb/storage.rs b/tinycloud-core/src/duckdb/storage.rs index 53ae3d467..b75256ed7 100644 --- a/tinycloud-core/src/duckdb/storage.rs +++ b/tinycloud-core/src/duckdb/storage.rs @@ -15,7 +15,17 @@ pub enum StorageMode { /// Open a database with the native diagnostic and external-access policy locked. /// A missing path selects an in-memory database. pub fn open_connection(path: Option<&Path>, max_memory: &str) -> Result { - open_guarded_connection(path, Some(max_memory)) + open_guarded_connection(path, Some(max_memory), 0) +} + +/// [`open_connection`], with at most `max_temp_directory_bytes` spilled to the +/// connection's temporary directory (0 keeps DuckDB's default). +pub fn open_spill_capped_connection( + path: Option<&Path>, + max_memory: &str, + max_temp_directory_bytes: u64, +) -> Result { + open_guarded_connection(path, Some(max_memory), max_temp_directory_bytes) } /// Open an offline snapshot/cache under the node's complete native admission @@ -23,12 +33,13 @@ pub fn open_connection(path: Option<&Path>, max_memory: &str) -> Result Result { - open_guarded_connection(Some(path), None) + open_guarded_connection(Some(path), None, 0) } fn open_guarded_connection( path: Option<&Path>, max_memory: Option<&str>, + max_temp_directory_bytes: u64, ) -> Result { // External access must already be disabled during native open/WAL recovery. // Profiling and logging are disabled by default; their setters require a @@ -49,7 +60,7 @@ fn open_guarded_connection( } }; - apply_security_settings(&conn, max_memory)?; + apply_security_settings(&conn, max_memory, max_temp_directory_bytes)?; audit_stored_code(&conn)?; Ok(conn) @@ -75,7 +86,11 @@ fn validate_max_memory(value: &str) -> Result<(), DuckDbError> { Ok(()) } -fn apply_security_settings(conn: &Connection, max_memory: Option<&str>) -> Result<(), DuckDbError> { +fn apply_security_settings( + conn: &Connection, + max_memory: Option<&str>, + max_temp_directory_bytes: u64, +) -> Result<(), DuckDbError> { if let Some(max_memory) = max_memory { validate_max_memory(max_memory)?; } @@ -107,6 +122,16 @@ fn apply_security_settings(conn: &Connection, max_memory: Option<&str>) -> Resul DuckDbError::Internal(format!("Failed to apply security settings: {}", e)) })?; } + if max_temp_directory_bytes > 0 { + let sql = format!("SET max_temp_directory_size = '{max_temp_directory_bytes}B';"); + let result = { + let _timer = QueryTimer::new(); + conn.execute_batch(&sql) + }; + result.map_err(|e| { + DuckDbError::Internal(format!("Failed to apply security settings: {}", e)) + })?; + } let result = { let _timer = QueryTimer::new(); conn.execute_batch("SET lock_configuration = true;") @@ -228,7 +253,18 @@ pub fn promote_to_file( path: &Path, max_memory: &str, ) -> Result { - let file_conn = open_connection(Some(path), max_memory)?; + promote_to_spill_capped_file(conn, path, max_memory, 0) +} + +/// [`promote_to_file`], opening the file under the spill limit of +/// [`open_spill_capped_connection`]. +pub fn promote_to_spill_capped_file( + conn: &Connection, + path: &Path, + max_memory: &str, + max_temp_directory_bytes: u64, +) -> Result { + let file_conn = open_spill_capped_connection(Some(path), max_memory, max_temp_directory_bytes)?; // Copy tables from in-memory conn to file-backed conn copy_tables(conn, &file_conn)?; diff --git a/tinycloud-core/src/duckdb/types.rs b/tinycloud-core/src/duckdb/types.rs index 5fd4df17a..cdcfc0458 100644 --- a/tinycloud-core/src/duckdb/types.rs +++ b/tinycloud-core/src/duckdb/types.rs @@ -470,6 +470,8 @@ pub enum DuckDbError { /// not loaded. #[error("sql_database_size_limit: the database is larger than this node loads; no change was applied")] DatabaseTooLargeToLoad, + #[error("duckdb_disk_headroom: The node is short of disk space for this write")] + DiskHeadroom, #[error("Database artifact replacement is temporarily unavailable")] Unavailable, #[error("Invalid statement: {0}")] diff --git a/tinycloud-node-server/src/config.rs b/tinycloud-node-server/src/config.rs index 53e61ab95..4624fba0c 100644 --- a/tinycloud-node-server/src/config.rs +++ b/tinycloud-node-server/src/config.rs @@ -1779,6 +1779,50 @@ pub struct DuckDbStorageConfig { pub idle_timeout_secs: u64, #[serde(default = "default_duckdb_max_memory")] pub max_memory_per_connection: String, + /// Free disk the node keeps behind the DuckDB cache (TC-2917); a write + /// or import short of it answers `507 duckdb_disk_headroom`. Unset: 2 GiB + /// when chunked artifact storage is enabled, otherwise 0 (off). The + /// 2 GiB default is provisional until the `storage_headroom` event shows + /// real headroom on production nodes. + #[serde(default)] + pub min_free_bytes: Option, + /// The most one DuckDB connection may spill to its temporary directory. + /// Unset: 1 GiB when chunked artifact storage is enabled, otherwise + /// DuckDB's own default. 0 keeps DuckDB's default. + #[serde(default)] + pub max_temp_directory_size: Option, +} + +/// Flag-on default of `storage.duckdb.min_free_bytes` (TC-2917, provisional). +#[cfg(feature = "duckdb")] +const CHUNKED_DUCKDB_MIN_FREE_BYTES: u64 = 2 << 30; +/// Flag-on default of `storage.duckdb.max_temp_directory_size` (TC-2917). +#[cfg(feature = "duckdb")] +const CHUNKED_DUCKDB_MAX_TEMP_DIRECTORY_BYTES: u64 = 1 << 30; + +#[cfg(feature = "duckdb")] +impl DuckDbStorageConfig { + /// The DuckDB disk policy. `chunked` is whether chunked artifact storage + /// is enabled, which sets the defaults; `budget` is the node-wide disk + /// budget shared with SQL. + pub fn disk_policy( + &self, + chunked: bool, + budget: std::sync::Arc, + ) -> tinycloud_core::duckdb::disk::DuckDbDiskPolicy { + let or_flag_default = |value: Option, default: u64| { + value.map_or(if chunked { default } else { 0 }, |value| value.as_u64()) + }; + tinycloud_core::duckdb::disk::DuckDbDiskPolicy { + min_free_bytes: or_flag_default(self.min_free_bytes, CHUNKED_DUCKDB_MIN_FREE_BYTES), + max_temp_directory_bytes: or_flag_default( + self.max_temp_directory_size, + CHUNKED_DUCKDB_MAX_TEMP_DIRECTORY_BYTES, + ), + budget, + ..Default::default() + } + } } fn default_duckdb_memory_threshold() -> ByteUnit { @@ -1800,6 +1844,8 @@ impl Default for DuckDbStorageConfig { memory_threshold: default_duckdb_memory_threshold(), idle_timeout_secs: default_duckdb_idle_timeout(), max_memory_per_connection: default_duckdb_max_memory(), + min_free_bytes: None, + max_temp_directory_size: None, } } } @@ -2251,6 +2297,31 @@ mod tests { use std::io::Write; use tempfile::NamedTempFile; + #[cfg(feature = "duckdb")] + #[test] + fn duckdb_disk_bounds_default_on_only_with_chunked_storage() { + let budget = std::sync::Arc::new(tinycloud_core::sql::durability::DiskBudget::default()); + let unset = DuckDbStorageConfig::default(); + let on = unset.disk_policy(true, budget.clone()); + assert_eq!( + (on.min_free_bytes, on.max_temp_directory_bytes), + (2 << 30, 1 << 30) + ); + let off = unset.disk_policy(false, budget.clone()); + assert_eq!((off.min_free_bytes, off.max_temp_directory_bytes), (0, 0)); + let set = DuckDbStorageConfig { + min_free_bytes: Some(ByteUnit::Byte(0)), + max_temp_directory_size: Some(ByteUnit::Mebibyte(256)), + ..Default::default() + }; + let explicit = set.disk_policy(true, budget.clone()); + assert_eq!( + (explicit.min_free_bytes, explicit.max_temp_directory_bytes), + (0, 256 << 20) + ); + assert!(std::sync::Arc::ptr_eq(&explicit.budget, &budget)); + } + #[test] fn column_encryption_strict_defaults_off_and_accepts_opt_in() { assert!(!Config::default().column_encryption.strict); diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index c62540602..e4e4df87b 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -1055,13 +1055,17 @@ pub async fn app_with_control( ) .await?; + // One disk budget for the node: SQL and DuckDB writes reserve from it. + let sql_durability = tinycloud_config.storage.sql.durability(); + #[cfg(feature = "duckdb")] + let disk_budget = sql_durability.budget.clone(); let sql_service = SqlService::new( tinycloud_config.storage.sql.path.clone().expect("resolved"), tinycloud_config.storage.sql.memory_threshold.as_u64(), database_artifact_repository.clone(), ) .with_wait_bounds(tinycloud_config.storage.sql.wait_bounds()) - .with_durability(tinycloud_config.storage.sql.durability()) + .with_durability(sql_durability) .with_ddl_policy(tinycloud_config.storage.sql.ddl_requires_schema); let mut policy_v3_runtime = @@ -1108,7 +1112,15 @@ pub async fn app_with_control( .max_memory_per_connection .clone(), database_artifact_repository, + ) + .with_disk_policy( + tinycloud_config + .storage + .duckdb + .disk_policy(artifact_collector.is_some(), disk_budget), ); + #[cfg(feature = "duckdb")] + duckdb_service.sweep_stale_cache_files().await; // Billing shares TINYCLOUD_ADMIN_SECRET with the node (it calls the // node's admin API with it) and accepts it on quota fetches (TC-627). diff --git a/tinycloud-node-server/src/prometheus.rs b/tinycloud-node-server/src/prometheus.rs index f612811ae..63639f1b5 100644 --- a/tinycloud-node-server/src/prometheus.rs +++ b/tinycloud-node-server/src/prometheus.rs @@ -206,9 +206,10 @@ lazy_static! { &["store"] ) .unwrap(); - /// Requests answered `507 sql_disk_headroom` (any SQL request whose error - /// maps to it: writes, and exports or queries that hit a full disk). The - /// label is the engine (`sql`). + /// Requests answered `507 sql_disk_headroom` or `507 duckdb_disk_headroom` + /// (any request whose error maps to it: writes, and exports or queries + /// that hit a full disk or a spill limit). The label is the engine (`sql`, + /// `duckdb`). pub static ref DISK_HEADROOM_REFUSALS: IntCounterVec = register_int_counter_vec!( "tinycloud_disk_headroom_refusals_total", "Requests refused for disk headroom, by engine.", diff --git a/tinycloud-node-server/src/routes/mod.rs b/tinycloud-node-server/src/routes/mod.rs index 3d9759e0b..a6856a0f1 100644 --- a/tinycloud-node-server/src/routes/mod.rs +++ b/tinycloud-node-server/src/routes/mod.rs @@ -4164,6 +4164,10 @@ fn duckdb_error_to_status(err: &DuckDbError) -> Status { DuckDbError::QuotaExceeded => Status::new(429), DuckDbError::StorageWouldGrow => Status::new(402), DuckDbError::DatabaseSizeLimit | DuckDbError::DatabaseTooLargeToLoad => Status::new(507), + DuckDbError::DiskHeadroom => { + crate::prometheus::observe_disk_headroom_refusal("duckdb"); + Status::new(507) + } DuckDbError::Unavailable => Status::ServiceUnavailable, DuckDbError::IngestError(_) => Status::InternalServerError, DuckDbError::ExportError(_) => Status::InternalServerError, @@ -8456,6 +8460,18 @@ mod tests { ); } + #[cfg(feature = "duckdb")] + #[test] + fn duckdb_disk_headroom_is_507_with_its_own_code() { + assert_eq!( + duckdb_error_to_status(&DuckDbError::DiskHeadroom), + Status::new(507) + ); + assert!(DuckDbError::DiskHeadroom + .to_string() + .starts_with("duckdb_disk_headroom:")); + } + #[tokio::test] async fn sql_request_is_write_classifies_variants() { use tinycloud_core::sql::types::SqlStatement; diff --git a/tinycloud.toml b/tinycloud.toml index 31de6ff75..6258c6247 100644 --- a/tinycloud.toml +++ b/tinycloud.toml @@ -42,6 +42,11 @@ cors = true # ddl_requires_schema = "enforce" # [global.storage.duckdb] # path = "./data/duckdb" + ## DuckDB disk bounds (TC-2917). Unset, both default on only with chunked + ## artifact storage: 2 GiB kept free (provisional) and 1 GiB of spill per + ## connection; otherwise off. 0 turns either off. + # min_free_bytes = "2 GiB" # short of it: 507 duckdb_disk_headroom + # max_temp_directory_size = "1 GiB" # spill past it: 507 duckdb_disk_headroom [global.storage.blocks] type = "Local" From 32cdd0679b07385f6a1644333be79d0325cf4560 Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:53:36 +0000 Subject: [PATCH 2/3] fix(TC-2917): DuckDB headroom refusals keep nothing; writes classified by the parser Review of #603: a write refused for headroom drops the database's local copy, so a COMMIT interrupted after it committed (or earlier statements of a batch) is never kept or saved later; a query whose SQL writes reserves and runs under the watcher; the watcher keeps interrupting while free space is short; the startup sweep does not descend into linked space directories; with chunked artifacts the node refuses a DuckDB floor below the artifact ceiling. Tests adopted from the review. --- CHANGELOG.md | 2 +- docs/operations/chunked-sql-artifacts.md | 20 +- tinycloud-core/src/duckdb/database.rs | 2 +- tinycloud-core/src/duckdb/disk.rs | 46 ++- tinycloud-core/src/duckdb/service.rs | 461 ++++++++++++++++++++++- tinycloud-node-server/src/config.rs | 28 +- tinycloud-node-server/src/lib.rs | 8 + 7 files changed, 533 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea20a28ab..5ebbc4b16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Policy/v3: minting a session and registering a policy no longer re-verify signatures or re-run the claim, presentation, credential admission and allow evaluations while holding their spaces' write order and row locks. Under the locks they check that the challenge, registration and roots are unchanged; those decisions were made before the locks, at the same time, from the same inputs. Refusal codes are unchanged, except that `credential-evaluation-changed` and `policy-evaluation-changed` can no longer occur. Details: TC-2986. - Artifact-part writers prioritize their own predecessor reclamation, keep replacement contention retryable, and distinguish capacity refusals from space-growth limits; existing database layouts can checkpoint before adopting WAL parts. Details: TC-2917. - DuckDB cache cleanup no longer holds up lookups of other databases while files are deleted; a spill directory left by a stopped process no longer blocks hydration or discard, and a failed import leaves no staged or imported file behind. Details: TC-2917. -- DuckDB keeps disk headroom: a write or import that would leave less than `storage.duckdb.min_free_bytes` free, a write still running when free space drops below it, and a statement that spills past `storage.duckdb.max_temp_directory_size` answer `507 duckdb_disk_headroom` and are never applied or acknowledged; each is counted as `tinycloud_disk_headroom_refusals_total{engine="duckdb"}`. Reservations share the node-wide disk budget with SQL. Unset, the floor is 2 GiB (provisional until production headroom is measured) and the spill limit 1 GiB per connection when chunked artifact storage is enabled; otherwise both are off. Details: TC-2917. +- DuckDB keeps disk headroom: a write or import that would leave less than `storage.duckdb.min_free_bytes` free, a write still running when free space drops below it, and a statement that spills past `storage.duckdb.max_temp_directory_size` answer `507 duckdb_disk_headroom`; each is counted as `tinycloud_disk_headroom_refusals_total{engine="duckdb"}`. A write refused this way keeps nothing: the database's local copy is dropped and reloaded from its last saved artifact, so neither a transaction that committed just before the refusal nor earlier statements of its batch are kept or saved later. A query whose SQL writes counts as a write. Reservations share the node-wide disk budget with SQL. Unset, the floor is 2 GiB (provisional until production headroom is measured) and the spill limit 1 GiB per connection when chunked artifact storage is enabled; otherwise both are off. With chunked artifacts enabled, the node refuses to start when the DuckDB floor is below `storage.sql.chunked_artifact_max_bytes`. Details: TC-2917. - At startup the node removes DuckDB spill directories and staging or renamed-aside cache files left by a stopped process, before it serves; the cache files themselves stay. Details: TC-2917. - DuckDB saves a complete checkpoint when a write changes its main file, and uses incremental WAL persistence only when that file still matches the stored checkpoint, so acknowledged writes remain recoverable after restart. Details: TC-2990. - Opt-in SQL artifact parts also stream cumulative WAL updates, retaining the checkpoint until a full checkpoint is needed. Details: TC-2917. diff --git a/docs/operations/chunked-sql-artifacts.md b/docs/operations/chunked-sql-artifacts.md index 5fc0a1ea9..54a8dfdd9 100644 --- a/docs/operations/chunked-sql-artifacts.md +++ b/docs/operations/chunked-sql-artifacts.md @@ -146,19 +146,27 @@ enabled, `storage.duckdb.min_free_bytes` defaults to 2 GiB and `storage.duckdb.max_temp_directory_size` to 1 GiB per connection; with parts off both stay off unless set. The 2 GiB floor is provisional until the `storage_headroom` event shows real headroom on production nodes; operators tune -it by config. +it by config. With parts enabled, the node refuses to start unless the floor is +at least `storage.sql.chunked_artifact_max_bytes`: a write's save copies the +database once more, outside the write's reservation. Leave a margin above that. +- A write is any request that can change the database: executes, batches, + imports, and a query whose SQL is not read-only (the parser decides, as for + authorization). - Before a write runs, it reserves its local database files' size (the bound of the snapshot copy its save can take); an import reserves twice its size. Reservations share the node-wide budget with SQL. A reservation that would leave less than the floor is refused with `507 duckdb_disk_headroom` and nothing runs. -- While a write runs, it is interrupted if free space falls below what other - operations hold plus the floor. The write is not applied, not saved and not - acknowledged; it answers `507 duckdb_disk_headroom`. +- While a write runs, it is interrupted whenever free space is below what + other operations hold plus the floor. It answers `507 duckdb_disk_headroom`, + and the database's local copy is dropped and reloaded from its last saved + artifact on the next request. So nothing from the request is kept: not a + transaction that committed just before the interrupt was reported, and not + the earlier statements of a non-transactional batch. - A statement that spills past the per-connection limit also answers - `507 duckdb_disk_headroom`. Queries and exports reserve nothing and are not - interrupted for headroom; only the spill limit applies to them. + `507 duckdb_disk_headroom`. Read-only queries and exports reserve nothing and + are not interrupted for headroom; only the spill limit applies to them. - Hydration is not gated: it writes at most the artifact ceiling and is freed at eviction. - At startup, before it serves, the node removes what a stopped process left in diff --git a/tinycloud-core/src/duckdb/database.rs b/tinycloud-core/src/duckdb/database.rs index 00fd54a88..c8706f9d0 100644 --- a/tinycloud-core/src/duckdb/database.rs +++ b/tinycloud-core/src/duckdb/database.rs @@ -370,7 +370,7 @@ pub(crate) fn spawn_actor_with_disk( response_tx, } => { let budget_key = disk::budget_key(&space_id, &db_name); - let write = disk::writes(&request); + let write = disk::writes(&request, &caveats, &ability); let result = disk.watch(&budget_key, &file_path, &conn, write, || { if without_growth { handle_message_without_growth( diff --git a/tinycloud-core/src/duckdb/disk.rs b/tinycloud-core/src/duckdb/disk.rs index ecffc7b75..e10c21b1c 100644 --- a/tinycloud-core/src/duckdb/disk.rs +++ b/tinycloud-core/src/duckdb/disk.rs @@ -4,9 +4,10 @@ //! cache. Before a write or an import runs, the bytes it can need are reserved //! in the node-wide [`DiskBudget`] that SQL uses too, so two databases cannot //! count the same free bytes. While a statement runs, a watcher interrupts it -//! once free space falls below what other operations hold plus the floor. A -//! refused or interrupted write answers [`DuckDbError::DiskHeadroom`] and is -//! never acknowledged. +//! whenever free space is below what other operations hold plus the floor. A +//! refused or interrupted write answers [`DuckDbError::DiskHeadroom`], is never +//! acknowledged, and the database's local state is discarded so nothing from +//! the request is kept or saved later. use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; @@ -17,6 +18,7 @@ use crate::sql::durability::{ DiskBudget, FreeSpace, ReservationGuard, ReservationRefused, StatvfsFreeSpace, }; +use super::caveats::DuckDbCaveats; use super::types::{DuckDbError, DuckDbRequest}; /// How often the watcher reads free space while a statement runs. @@ -62,16 +64,21 @@ pub(crate) fn budget_key(space: &str, db_name: &str) -> String { } /// Requests that can change the database and are saved afterwards: they -/// reserve room and run under the watcher. Queries, exports and `describe` -/// do neither; only the spill limit applies to them. -pub(crate) fn writes(request: &DuckDbRequest) -> bool { - !matches!( - request, - DuckDbRequest::Query { .. } - | DuckDbRequest::Describe - | DuckDbRequest::Export - | DuckDbRequest::ExportToKv { .. } - ) +/// reserve room and run under the watcher. A `query` counts when its SQL is +/// not read-only (the parser decides, as the route does); exports and +/// `describe` never do. Only the spill limit applies to the rest. +pub(crate) fn writes( + request: &DuckDbRequest, + caveats: &Option, + ability: &str, +) -> bool { + match request { + // A query the parser refuses never runs; counting it is harmless. + DuckDbRequest::Query { sql, .. } => super::parser::validate_sql(sql, caveats, ability) + .map_or(true, |parsed| !parsed.is_read_only), + DuckDbRequest::Describe | DuckDbRequest::Export | DuckDbRequest::ExportToKv { .. } => false, + _ => true, + } } impl DuckDbDiskPolicy { @@ -110,10 +117,14 @@ impl DuckDbDiskPolicy { } } - /// Run `statement` on `conn`. A write (`write`) is interrupted if free - /// space behind `dir` falls below what other operations hold plus the - /// floor. A statement stopped that way, or one that hit the spill limit, - /// is [`DuckDbError::DiskHeadroom`]. + /// Run `statement` on `conn`. A write (`write`) is interrupted whenever + /// free space behind `dir` is below what other operations hold plus the + /// floor: DuckDB clears an interrupt when its next query starts, so the + /// watcher re-interrupts on every short reading until the statement + /// returns. A failed statement that was interrupted, or one that hit the + /// spill limit, is [`DuckDbError::DiskHeadroom`]; the caller discards the + /// database's local state, because a COMMIT can finish before DuckDB + /// reports the interrupt. pub(crate) fn watch( &self, key: &str, @@ -147,7 +158,6 @@ impl DuckDbDiskPolicy { if short { tripped.store(true, Ordering::Relaxed); interrupt.interrupt(); - break; } } }); diff --git a/tinycloud-core/src/duckdb/service.rs b/tinycloud-core/src/duckdb/service.rs index b9ba82c75..d58956172 100644 --- a/tinycloud-core/src/duckdb/service.rs +++ b/tinycloud-core/src/duckdb/service.rs @@ -108,6 +108,11 @@ impl DuckDbService { return 0; }; for space in spaces.filter_map(Result::ok) { + // `file_type` does not follow links: a link in place of a + // space directory is never descended into. + if !space.file_type().is_ok_and(|kind| kind.is_dir()) { + continue; + } let Ok(entries) = std::fs::read_dir(space.path()) else { continue; }; @@ -204,7 +209,8 @@ impl DuckDbService { // Room for this write and for the snapshot copy its save can take, // which is at most the local database files. - let _reservation = if disk::writes(&request) { + let writes = disk::writes(&request, &caveats, &ability); + let _reservation = if writes { let cache_path = self.cache_path(space, db_name); let local_bytes = local_file_bytes(&cache_path).await; self.disk @@ -222,6 +228,14 @@ impl DuckDbService { { self.discard_local_state(&key).await?; } + // A write stopped for disk headroom may have committed locally (an + // interrupted COMMIT reports the interrupt after committing), or + // applied earlier statements of its batch. Drop the local copy, so + // the durable artifact stays the truth and no later save carries the + // refused request. + if writes && matches!(result, Err(DuckDbError::DiskHeadroom)) { + self.discard_local_state(&key).await?; + } let result = result?; if !result.write_targets.is_empty() { @@ -2924,4 +2938,449 @@ mod tests { ] ); } + + /// A link in place of a space directory is never descended into. + #[cfg(unix)] + #[tokio::test] + async fn duckdb_startup_sweep_does_not_follow_a_linked_space() { + let cache = TempDir::new().unwrap(); + let outside = TempDir::new().unwrap(); + let service = service(&cache, artifact_repository().await); + std::fs::create_dir_all(outside.path().join("x.duckdb.tmp")).unwrap(); + std::fs::write(outside.path().join("x.duckdb.tmp.1.2"), b"x").unwrap(); + std::os::unix::fs::symlink(outside.path(), cache.path().join("space-link")).unwrap(); + assert_eq!(service.sweep_stale_cache_files().await, 0); + assert_eq!( + dir_entries(outside.path()), + vec!["x.duckdb.tmp", "x.duckdb.tmp.1.2"] + ); + } + + mod headroom { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering::SeqCst}; + use std::time::Duration; + + const MIN: u64 = 8 << 20; + const PLENTY: u64 = 64 << 20; + + async fn exec( + service: &DuckDbService, + space: &SpaceId, + db: &str, + request: DuckDbRequest, + ) -> Result { + service + .execute( + space, + db, + request, + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await + } + + async fn count(service: &DuckDbService, space: &SpaceId, db: &str, sql: &str) -> i64 { + let result = service + .execute( + space, + db, + DuckDbRequest::Query { + sql: sql.into(), + params: vec![], + }, + None, + "tinycloud.duckdb/read".into(), + false, + ) + .await + .unwrap(); + let DuckDbResponse::Query(rows) = result.response else { + panic!("not a query response"); + }; + match &rows.rows[0][0] { + DuckDbValue::Integer(n) => *n, + other => panic!("{other:?}"), + } + } + + fn statement(sql: &str) -> DuckDbStatement { + DuckDbStatement { + sql: sql.into(), + params: vec![], + } + } + + fn short_after(free: &Arc, after: Duration) -> std::thread::JoinHandle<()> { + let free = free.clone(); + std::thread::spawn(move || { + std::thread::sleep(after); + free.0.store(0, SeqCst); + }) + } + + /// A repository with chunked artifacts on: databases are file-backed + /// from their first write, as with the flag on. + async fn chunked( + test: &str, + ) -> Option<( + crate::test_support::PostgresTestSchema, + Arc, + )> { + let fixture = crate::test_support::PostgresTestSchema::new(test).await?; + let repo = Arc::new( + SeaOrmDatabaseArtifactRepository::new(fixture.conn.clone()).with_chunk_limits( + Some(crate::database_artifacts::chunks::ChunkLimits { + artifact_bytes: 1 << 30, + account_bytes: 4 << 30, + }), + ), + ); + Some((fixture, repo)) + } + + fn chunked_service( + cache: &TempDir, + repo: Arc, + ) -> DuckDbService { + DuckDbService::new( + cache.path().to_string_lossy().into_owned(), + u64::MAX, + 300, + "1GB".into(), + repo, + ) + } + + /// A write sent as `query` is held to the floor like any other write. + #[tokio::test] + async fn postgres_duckdb_query_shaped_write_is_held_to_the_floor() { + let Some((_fixture, repo)) = chunked("postgres_duckdb_query_shaped_write_floor").await + else { + return; + }; + let cache = TempDir::new().unwrap(); + let free = Arc::new(FakeFree(PLENTY.into())); + let service = + chunked_service(&cache, repo.clone()).with_disk_policy(disk_policy(&free, MIN)); + let space = test_space_id("query-shaped-write"); + exec( + &service, + &space, + "main", + write_request("INSERT INTO t VALUES (1)", Some("CREATE TABLE t(i BIGINT)")), + ) + .await + .unwrap(); + + free.0.store(0, SeqCst); + let query = exec( + &service, + &space, + "main", + DuckDbRequest::Query { + sql: "INSERT INTO t VALUES (3)".into(), + params: vec![], + }, + ) + .await; + assert!( + matches!(query, Err(DuckDbError::DiskHeadroom)), + "{:?}", + query.map(|_| ()) + ); + + // Free space running out while a long query-shaped write runs. + free.0.store(PLENTY, SeqCst); + let flip = short_after(&free, Duration::from_millis(200)); + let long = exec( + &service, + &space, + "main", + DuckDbRequest::Query { + sql: "INSERT INTO t SELECT i FROM range(8000000) r(i) WHERE md5(i::VARCHAR) <> ''" + .into(), + params: vec![], + }, + ) + .await; + flip.join().unwrap(); + assert!( + matches!(long, Err(DuckDbError::DiskHeadroom)), + "{:?}", + long.map(|_| ()) + ); + + free.0.store(PLENTY, SeqCst); + assert_eq!( + count(&service, &space, "main", "SELECT count(*) FROM t").await, + 1 + ); + let fresh = TempDir::new().unwrap(); + let durable = chunked_service(&fresh, repo); + assert_eq!( + count(&durable, &space, "main", "SELECT count(*) FROM t").await, + 1 + ); + } + + /// A request refused for headroom keeps nothing: not the statements of + /// its batch that ran before the refusal, locally or after the next + /// saved write. + #[tokio::test] + async fn duckdb_headroom_refusal_keeps_nothing_from_the_request() { + let repo = artifact_repository().await; + let cache = TempDir::new().unwrap(); + let free = Arc::new(FakeFree(PLENTY.into())); + let service = + file_service(&cache, repo.clone()).with_disk_policy(disk_policy(&free, MIN)); + let space = test_space_id("headroom-keeps-nothing"); + exec( + &service, + &space, + "main", + write_request( + "CREATE TABLE m(k VARCHAR)", + Some("CREATE TABLE t(i BIGINT)"), + ), + ) + .await + .unwrap(); + let flip = short_after(&free, Duration::from_millis(300)); + let refused = exec( + &service, + &space, + "main", + DuckDbRequest::Batch { + statements: vec![ + statement("INSERT INTO m VALUES ('before-the-refusal')"), + statement( + "INSERT INTO t SELECT i FROM range(2000000000) r(i) WHERE md5(i::VARCHAR) <> ''", + ), + ], + transactional: false, + }, + ) + .await; + flip.join().unwrap(); + free.0.store(PLENTY, SeqCst); + assert!( + matches!(refused, Err(DuckDbError::DiskHeadroom)), + "{:?}", + refused.map(|_| ()) + ); + let kept = "SELECT count(*) FROM m WHERE k = 'before-the-refusal'"; + assert_eq!(count(&service, &space, "main", kept).await, 0); + exec( + &service, + &space, + "main", + write_request("INSERT INTO m VALUES ('next')", None), + ) + .await + .unwrap(); + let fresh = TempDir::new().unwrap(); + let durable = file_service(&fresh, repo); + assert_eq!(count(&durable, &space, "main", kept).await, 0); + assert_eq!( + count(&durable, &space, "main", "SELECT count(*) FROM m").await, + 1 + ); + } + + /// Free space that runs short from a given watcher reading on, so a + /// refusal can be placed late in a write, around its COMMIT. + struct ShortFromReading { + readings: AtomicU64, + short_from: AtomicU64, + } + + impl crate::sql::durability::FreeSpace for ShortFromReading { + fn free_bytes(&self, _: &Path) -> std::io::Result { + let n = self.readings.fetch_add(1, SeqCst) + 1; + Ok(if n >= self.short_from.load(SeqCst) { + 0 + } else { + 1 << 36 + }) + } + } + + /// A transactional batch refused near or during its COMMIT is never + /// kept: an answer of 507 means the change is in neither the local + /// copy nor the durable artifact. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn postgres_duckdb_batch_refused_around_commit_is_not_kept() { + let Some((_fixture, repo)) = chunked("postgres_duckdb_batch_refused_commit").await + else { + return; + }; + let cache = TempDir::new().unwrap(); + let free = Arc::new(ShortFromReading { + readings: AtomicU64::new(0), + short_from: AtomicU64::new(u64::MAX), + }); + let service = + chunked_service(&cache, repo.clone()).with_disk_policy(DuckDbDiskPolicy { + min_free_bytes: MIN, + free_space: free.clone(), + ..Default::default() + }); + let space = test_space_id("batch-refused-commit"); + let setup = |db: String| { + let (service, space) = (&service, &space); + async move { + exec( + service, + space, + &db, + DuckDbRequest::Batch { + statements: vec![ + statement("CREATE TABLE t(i BIGINT, s VARCHAR)"), + statement("CREATE TABLE m(k VARCHAR)"), + statement( + "INSERT INTO t SELECT i, md5(i::VARCHAR) FROM range(1000000) r(i)", + ), + ], + transactional: false, + }, + ) + .await + .unwrap(); + } + }; + let batch = |marker: &str| DuckDbRequest::Batch { + statements: vec![ + statement(&format!("INSERT INTO m VALUES ('{marker}')")), + statement(&format!("UPDATE t SET s = md5(s || '{marker}')")), + ], + transactional: true, + }; + // How many free-space readings one batch takes. + setup("calibrate".into()).await; + let before = free.readings.load(SeqCst); + exec(&service, &space, "calibrate", batch("calibrate")) + .await + .unwrap(); + let readings = free.readings.load(SeqCst) - before; + let mut kept = Vec::new(); + for round in 0..10u64 { + let db = format!("round{round}"); + free.short_from.store(u64::MAX, SeqCst); + setup(db.clone()).await; + let marker = format!("x{round}"); + let start = free.readings.load(SeqCst); + free.short_from.store( + start + readings.saturating_sub(1 + 2 * round).max(2), + SeqCst, + ); + let result = exec(&service, &space, &db, batch(&marker)).await; + free.short_from.store(u64::MAX, SeqCst); + let marked = format!("SELECT count(*) FROM m WHERE k = '{marker}'"); + let local = count(&service, &space, &db, &marked).await; + if result.is_err() { + exec( + &service, + &space, + &db, + write_request("INSERT INTO m VALUES ('next')", None), + ) + .await + .unwrap(); + let fresh = TempDir::new().unwrap(); + let durable = + count(&chunked_service(&fresh, repo.clone()), &space, &db, &marked).await; + if local != 0 || durable != 0 { + kept.push((round, local, durable)); + } + } else if local != 1 { + kept.push((round, local, -1)); + } + } + assert!( + kept.is_empty(), + "refused batches kept, or accepted batches lost: {kept:?}" + ); + } + + /// A shortfall first seen between two DuckDB calls still stops the + /// statement that follows: DuckDB clears an interrupt when a query + /// starts, so the watcher keeps interrupting. + #[test] + fn duckdb_shortfall_between_calls_still_stops_the_next_statement() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("d.duckdb"); + let conn = storage::open_connection(Some(&path), "512MB").unwrap(); + let free = Arc::new(FakeFree(0.into())); + let result = disk_policy(&free, MIN).watch("duckdb:test/d", &path, &conn, true, || { + // Rust-side work between two DuckDB calls, where the first + // reading lands. + std::thread::sleep(Duration::from_millis(120)); + conn.execute_batch( + "CREATE TABLE big AS SELECT i, md5(i::VARCHAR) s FROM range(3000000) r(i)", + ) + .map_err(|error| DuckDbError::DuckDb(error.to_string())) + }); + assert!( + matches!(result, Err(DuckDbError::DiskHeadroom)), + "{result:?}" + ); + } + + /// A write reserves its database's local size against the node-wide + /// budget: while another operation holds room, a write that would + /// leave less than the floor is refused before it runs, and the same + /// write runs once that room is released. Nothing stays reserved. + #[tokio::test] + async fn postgres_duckdb_write_reserves_its_database_size() { + let Some((_fixture, repo)) = chunked("postgres_duckdb_write_reserves_size").await + else { + return; + }; + let cache = TempDir::new().unwrap(); + let free = Arc::new(FakeFree(PLENTY.into())); + let policy = disk_policy(&free, MIN); + let service = chunked_service(&cache, repo).with_disk_policy(policy.clone()); + let space = test_space_id("write-reserves-size"); + exec( + &service, + &space, + "main", + write_request( + "INSERT INTO t SELECT i, md5(i::VARCHAR) FROM range(200000) r(i)", + Some("CREATE TABLE t(i BIGINT, s VARCHAR)"), + ), + ) + .await + .unwrap(); + let dir = cache.path().join(space.to_string()); + let size: u64 = ["", ".wal"] + .iter() + .map(|ext| { + std::fs::metadata(dir.join(format!("main.duckdb{ext}"))) + .map(|m| m.len()) + .unwrap_or(0) + }) + .sum(); + assert!(size > 0, "the database is not file-backed"); + // Room above the floor for one database of this size, not two. + free.0.store(MIN + size + size / 2, SeqCst); + let in_flight = policy + .reserve("duckdb:other/in-flight", size, &dir) + .unwrap() + .expect("the check is on"); + let small_write = || write_request("INSERT INTO t VALUES (-1, 'x')", None); + let refused = exec(&service, &space, "main", small_write()).await; + assert!( + matches!(refused, Err(DuckDbError::DiskHeadroom)), + "{:?}", + refused.map(|_| ()) + ); + drop(in_flight); + exec(&service, &space, "main", small_write()).await.unwrap(); + assert_eq!(policy.budget.reserved(), 0); + } + } } diff --git a/tinycloud-node-server/src/config.rs b/tinycloud-node-server/src/config.rs index 4624fba0c..34b9dbe1a 100644 --- a/tinycloud-node-server/src/config.rs +++ b/tinycloud-node-server/src/config.rs @@ -1802,6 +1802,19 @@ const CHUNKED_DUCKDB_MAX_TEMP_DIRECTORY_BYTES: u64 = 1 << 30; #[cfg(feature = "duckdb")] impl DuckDbStorageConfig { + /// The DuckDB free-space floor in effect: the configured value, else the + /// flag-on default when chunked artifact storage is enabled, else 0 (off). + pub fn effective_min_free_bytes(&self, chunked: bool) -> u64 { + self.min_free_bytes.map_or( + if chunked { + CHUNKED_DUCKDB_MIN_FREE_BYTES + } else { + 0 + }, + |value| value.as_u64(), + ) + } + /// The DuckDB disk policy. `chunked` is whether chunked artifact storage /// is enabled, which sets the defaults; `budget` is the node-wide disk /// budget shared with SQL. @@ -1810,14 +1823,15 @@ impl DuckDbStorageConfig { chunked: bool, budget: std::sync::Arc, ) -> tinycloud_core::duckdb::disk::DuckDbDiskPolicy { - let or_flag_default = |value: Option, default: u64| { - value.map_or(if chunked { default } else { 0 }, |value| value.as_u64()) - }; tinycloud_core::duckdb::disk::DuckDbDiskPolicy { - min_free_bytes: or_flag_default(self.min_free_bytes, CHUNKED_DUCKDB_MIN_FREE_BYTES), - max_temp_directory_bytes: or_flag_default( - self.max_temp_directory_size, - CHUNKED_DUCKDB_MAX_TEMP_DIRECTORY_BYTES, + min_free_bytes: self.effective_min_free_bytes(chunked), + max_temp_directory_bytes: self.max_temp_directory_size.map_or( + if chunked { + CHUNKED_DUCKDB_MAX_TEMP_DIRECTORY_BYTES + } else { + 0 + }, + |value| value.as_u64(), ), budget, ..Default::default() diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index e4e4df87b..2e8d9738c 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -621,6 +621,14 @@ pub fn resolve_runtime_config(tinycloud_config: &Config) -> Result { && resolved.storage.sql.chunked_account_max_bytes <= i64::MAX as u64, "chunked artifacts require explicit positive capacities within int64" ); + // A DuckDB write's save copies the database once more, outside the + // write's reservation, so the floor must hold one artifact ceiling. + #[cfg(feature = "duckdb")] + anyhow::ensure!( + resolved.storage.duckdb.effective_min_free_bytes(true) + >= resolved.storage.sql.chunked_artifact_max_bytes, + "storage.duckdb.min_free_bytes must be at least storage.sql.chunked_artifact_max_bytes when chunked artifacts are enabled" + ); } resolved From a485ceb9bb160286e065c8ad60e0a442de402b2b Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:22:29 +0000 Subject: [PATCH 3/3] fix(TC-2917): only plain queries skip the DuckDB disk floor; floor check test Review of #603 (32cdd067): a query counts as read-only for the disk floor only when every statement is a plain query or an EXPLAIN that does not run its statement, so an admin's PREPARE/EXECUTE is held to the floor; requests are not classified when no disk limit is set; the limit defaults key on the configured flag, like the startup check, which now has a test; docs say the floor cannot be 0 while parts are on and that refused SQL answers 507 below the floor. --- CHANGELOG.md | 2 +- docs/operations/chunked-sql-artifacts.md | 25 +++++---- tinycloud-core/src/duckdb/database.rs | 3 +- tinycloud-core/src/duckdb/disk.rs | 31 ++++++++--- tinycloud-core/src/duckdb/service.rs | 66 +++++++++++++++++++++++- tinycloud-node-server/src/config.rs | 50 ++++++++++++++++++ tinycloud-node-server/src/lib.rs | 3 +- 7 files changed, 160 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ebbc4b16..5356202df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Policy/v3: minting a session and registering a policy no longer re-verify signatures or re-run the claim, presentation, credential admission and allow evaluations while holding their spaces' write order and row locks. Under the locks they check that the challenge, registration and roots are unchanged; those decisions were made before the locks, at the same time, from the same inputs. Refusal codes are unchanged, except that `credential-evaluation-changed` and `policy-evaluation-changed` can no longer occur. Details: TC-2986. - Artifact-part writers prioritize their own predecessor reclamation, keep replacement contention retryable, and distinguish capacity refusals from space-growth limits; existing database layouts can checkpoint before adopting WAL parts. Details: TC-2917. - DuckDB cache cleanup no longer holds up lookups of other databases while files are deleted; a spill directory left by a stopped process no longer blocks hydration or discard, and a failed import leaves no staged or imported file behind. Details: TC-2917. -- DuckDB keeps disk headroom: a write or import that would leave less than `storage.duckdb.min_free_bytes` free, a write still running when free space drops below it, and a statement that spills past `storage.duckdb.max_temp_directory_size` answer `507 duckdb_disk_headroom`; each is counted as `tinycloud_disk_headroom_refusals_total{engine="duckdb"}`. A write refused this way keeps nothing: the database's local copy is dropped and reloaded from its last saved artifact, so neither a transaction that committed just before the refusal nor earlier statements of its batch are kept or saved later. A query whose SQL writes counts as a write. Reservations share the node-wide disk budget with SQL. Unset, the floor is 2 GiB (provisional until production headroom is measured) and the spill limit 1 GiB per connection when chunked artifact storage is enabled; otherwise both are off. With chunked artifacts enabled, the node refuses to start when the DuckDB floor is below `storage.sql.chunked_artifact_max_bytes`. Details: TC-2917. +- DuckDB keeps disk headroom: a write or import that would leave less than `storage.duckdb.min_free_bytes` free, a write still running when free space drops below it, and a statement that spills past `storage.duckdb.max_temp_directory_size` answer `507 duckdb_disk_headroom`; each is counted as `tinycloud_disk_headroom_refusals_total{engine="duckdb"}`. A write refused this way keeps nothing: the database's local copy is dropped and reloaded from its last saved artifact, so neither a transaction that committed just before the refusal nor earlier statements of its batch are kept or saved later. Only plain queries and `EXPLAIN` without `ANALYZE` sent as `query` count as reads; anything else (including `PREPARE`/`EXECUTE`) counts as a write. Reservations share the node-wide disk budget with SQL. Both limits key on `storage.sql.chunked_artifacts`: unset, the floor is 2 GiB (provisional until production headroom is measured) and the spill limit 1 GiB per connection when it is on; otherwise both are off. With chunked artifacts on, the node refuses to start when the DuckDB floor is below `storage.sql.chunked_artifact_max_bytes`, so the floor cannot be 0. Details: TC-2917. - At startup the node removes DuckDB spill directories and staging or renamed-aside cache files left by a stopped process, before it serves; the cache files themselves stay. Details: TC-2917. - DuckDB saves a complete checkpoint when a write changes its main file, and uses incremental WAL persistence only when that file still matches the stored checkpoint, so acknowledged writes remain recoverable after restart. Details: TC-2990. - Opt-in SQL artifact parts also stream cumulative WAL updates, retaining the checkpoint until a full checkpoint is needed. Details: TC-2917. diff --git a/docs/operations/chunked-sql-artifacts.md b/docs/operations/chunked-sql-artifacts.md index 54a8dfdd9..017a3b539 100644 --- a/docs/operations/chunked-sql-artifacts.md +++ b/docs/operations/chunked-sql-artifacts.md @@ -141,18 +141,23 @@ checkpoint also needs an owned temporary copy while streaming, and old/new versions overlap in PostgreSQL until their deletions commit. PostgreSQL can reuse deleted storage after vacuum; relation files need not shrink. Size refusal occurs after native execution. -DuckDB keeps disk headroom on the filesystem behind its cache. With parts -enabled, `storage.duckdb.min_free_bytes` defaults to 2 GiB and -`storage.duckdb.max_temp_directory_size` to 1 GiB per connection; with parts -off both stay off unless set. The 2 GiB floor is provisional until the -`storage_headroom` event shows real headroom on production nodes; operators tune -it by config. With parts enabled, the node refuses to start unless the floor is -at least `storage.sql.chunked_artifact_max_bytes`: a write's save copies the -database once more, outside the write's reservation. Leave a margin above that. +DuckDB keeps disk headroom on the filesystem behind its cache. Both limits key +on `storage.sql.chunked_artifacts`: with it on, `storage.duckdb.min_free_bytes` +defaults to 2 GiB and `storage.duckdb.max_temp_directory_size` to 1 GiB per +connection; with it off both stay off unless set. The 2 GiB floor is +provisional until the `storage_headroom` event shows real headroom on +production nodes; operators tune it by config. With chunked artifacts on, the +node refuses to start unless the floor is at least +`storage.sql.chunked_artifact_max_bytes`, so the floor cannot be turned off +(0) while they are on: a write's save copies the database once more, outside +the write's reservation. Leave a margin above that. - A write is any request that can change the database: executes, batches, - imports, and a query whose SQL is not read-only (the parser decides, as for - authorization). + imports, and any query that is not purely plain `SELECT`-style queries and + `EXPLAIN` without `ANALYZE` (so an admin's `PREPARE … AS INSERT` and + `EXECUTE`, `CALL`, `CREATE SCHEMA` and the like count). SQL that fails to + parse or is refused also counts, so while free space is below the floor it + answers `507 duckdb_disk_headroom` rather than its 400 or 403. - Before a write runs, it reserves its local database files' size (the bound of the snapshot copy its save can take); an import reserves twice its size. Reservations share the node-wide budget with SQL. A reservation that would diff --git a/tinycloud-core/src/duckdb/database.rs b/tinycloud-core/src/duckdb/database.rs index c8706f9d0..ef0bc931b 100644 --- a/tinycloud-core/src/duckdb/database.rs +++ b/tinycloud-core/src/duckdb/database.rs @@ -370,7 +370,8 @@ pub(crate) fn spawn_actor_with_disk( response_tx, } => { let budget_key = disk::budget_key(&space_id, &db_name); - let write = disk::writes(&request, &caveats, &ability); + let write = + disk.headroom_checked() && disk::writes(&request, &caveats, &ability); let result = disk.watch(&budget_key, &file_path, &conn, write, || { if without_growth { handle_message_without_growth( diff --git a/tinycloud-core/src/duckdb/disk.rs b/tinycloud-core/src/duckdb/disk.rs index e10c21b1c..bde10f94f 100644 --- a/tinycloud-core/src/duckdb/disk.rs +++ b/tinycloud-core/src/duckdb/disk.rs @@ -64,18 +64,31 @@ pub(crate) fn budget_key(space: &str, db_name: &str) -> String { } /// Requests that can change the database and are saved afterwards: they -/// reserve room and run under the watcher. A `query` counts when its SQL is -/// not read-only (the parser decides, as the route does); exports and -/// `describe` never do. Only the spill limit applies to the rest. +/// reserve room and run under the watcher. A `query` is read-only only when +/// the parser finds it read-only *and* every statement is a plain query or an +/// `EXPLAIN` that does not run its statement; anything else an admin may send +/// as a query (`PREPARE … AS INSERT` with `EXECUTE`, `CALL checkpoint()`, +/// `CREATE SCHEMA`, …) counts as a write, as does SQL the parser refuses. +/// Exports and `describe` never do. Only the spill limit applies to the rest. pub(crate) fn writes( request: &DuckDbRequest, caveats: &Option, ability: &str, ) -> bool { + use sqlparser::ast::Statement; match request { - // A query the parser refuses never runs; counting it is harmless. - DuckDbRequest::Query { sql, .. } => super::parser::validate_sql(sql, caveats, ability) - .map_or(true, |parsed| !parsed.is_read_only), + DuckDbRequest::Query { sql, .. } => !super::parser::validate_sql(sql, caveats, ability) + .is_ok_and(|parsed| { + parsed.is_read_only + && parsed.statements.iter().all(|statement| { + matches!( + statement, + Statement::Query(_) + | Statement::ExplainTable { .. } + | Statement::Explain { analyze: false, .. } + ) + }) + }), DuckDbRequest::Describe | DuckDbRequest::Export | DuckDbRequest::ExportToKv { .. } => false, _ => true, } @@ -86,6 +99,12 @@ impl DuckDbDiskPolicy { self.min_free_bytes > 0 } + /// Whether the policy can refuse anything: a floor or a spill limit. With + /// neither, requests are not classified at all. + pub(crate) fn active(&self) -> bool { + self.headroom_checked() || self.max_temp_directory_bytes > 0 + } + /// Reserve `bytes` for one operation of `key`, measured against the /// filesystem behind `dir`. `None` when the check is off. pub(crate) fn reserve( diff --git a/tinycloud-core/src/duckdb/service.rs b/tinycloud-core/src/duckdb/service.rs index d58956172..c5e5318da 100644 --- a/tinycloud-core/src/duckdb/service.rs +++ b/tinycloud-core/src/duckdb/service.rs @@ -209,7 +209,7 @@ impl DuckDbService { // Room for this write and for the snapshot copy its save can take, // which is at most the local database files. - let writes = disk::writes(&request, &caveats, &ability); + let writes = self.disk.active() && disk::writes(&request, &caveats, &ability); let _reservation = if writes { let cache_path = self.cache_path(space, db_name); let local_bytes = local_file_bytes(&cache_path).await; @@ -3126,6 +3126,70 @@ mod tests { ); } + /// An admin's `PREPARE … AS INSERT` and `EXECUTE`, sent as queries, + /// are held to the floor like the INSERT itself: the parser calls + /// both read-only, but neither is a plain query. + #[tokio::test] + async fn postgres_duckdb_prepared_insert_is_held_to_the_floor() { + let Some((_fixture, repo)) = chunked("postgres_duckdb_prepared_insert_floor").await + else { + return; + }; + let cache = TempDir::new().unwrap(); + let free = Arc::new(FakeFree(PLENTY.into())); + let service = chunked_service(&cache, repo).with_disk_policy(disk_policy(&free, MIN)); + let space = test_space_id("prepared-insert"); + exec( + &service, + &space, + "main", + write_request( + "INSERT INTO t VALUES ('x')", + Some("CREATE TABLE t(s VARCHAR)"), + ), + ) + .await + .unwrap(); + let query = |sql: &str| DuckDbRequest::Query { + sql: sql.into(), + params: vec![], + }; + free.0.store(0, SeqCst); + let prepare = exec( + &service, + &space, + "main", + query("PREPARE grow AS INSERT INTO t SELECT md5(i::VARCHAR) FROM range(1000) r(i)"), + ) + .await; + assert!( + matches!(prepare, Err(DuckDbError::DiskHeadroom)), + "{:?}", + prepare.map(|_| ()) + ); + free.0.store(PLENTY, SeqCst); + exec( + &service, + &space, + "main", + query("PREPARE grow AS INSERT INTO t SELECT md5(i::VARCHAR) FROM range(1000) r(i)"), + ) + .await + .unwrap(); + free.0.store(0, SeqCst); + let execute = exec(&service, &space, "main", query("EXECUTE grow")).await; + assert!( + matches!(execute, Err(DuckDbError::DiskHeadroom)), + "{:?}", + execute.map(|_| ()) + ); + free.0.store(PLENTY, SeqCst); + assert_eq!( + count(&service, &space, "main", "SELECT count(*) FROM t").await, + 1 + ); + } + /// A request refused for headroom keeps nothing: not the statements of /// its batch that ran before the refusal, locally or after the next /// saved write. diff --git a/tinycloud-node-server/src/config.rs b/tinycloud-node-server/src/config.rs index 34b9dbe1a..358184d99 100644 --- a/tinycloud-node-server/src/config.rs +++ b/tinycloud-node-server/src/config.rs @@ -2336,6 +2336,56 @@ mod tests { assert!(std::sync::Arc::ptr_eq(&explicit.budget, &budget)); } + /// With chunked artifacts on, the DuckDB floor must hold one artifact + /// ceiling; 0 (off) is refused too. With chunked artifacts off nothing is + /// required. + #[cfg(feature = "duckdb")] + #[test] + fn chunked_artifacts_require_a_duckdb_floor_of_one_artifact_ceiling() { + const GIB: u64 = 1 << 30; + let chunked = |ceiling: u64, floor: Option| { + let mut config = Config::default(); + config.storage.database = Some("postgres://test@localhost/test".into()); + config.storage.sql.chunked_artifacts = true; + config.storage.sql.chunked_artifact_max_bytes = ceiling; + config.storage.sql.chunked_account_max_bytes = ceiling * 4; + config.storage.duckdb.min_free_bytes = floor.map(ByteUnit::Byte); + config + }; + let mut flag_off = Config::default(); + flag_off.storage.duckdb.min_free_bytes = Some(ByteUnit::Byte(1)); + for (label, config, accepted) in [ + ( + "default floor (2 GiB), ceiling 2 GiB", + chunked(2 * GIB, None), + true, + ), + ( + "default floor (2 GiB), ceiling 2 GiB + 1", + chunked(2 * GIB + 1, None), + false, + ), + ( + "floor 4 GiB, ceiling 3 GiB", + chunked(3 * GIB, Some(4 * GIB)), + true, + ), + ( + "floor 1 GiB, ceiling 3 GiB", + chunked(3 * GIB, Some(GIB)), + false, + ), + ("floor 0 (off), ceiling 1 GiB", chunked(GIB, Some(0)), false), + ("flag off, floor 1 byte", flag_off, true), + ] { + assert_eq!( + crate::resolve_runtime_config(&config).is_ok(), + accepted, + "{label}" + ); + } + } + #[test] fn column_encryption_strict_defaults_off_and_accepts_opt_in() { assert!(!Config::default().column_encryption.strict); diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 2e8d9738c..544c94ad8 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -1121,11 +1121,12 @@ pub async fn app_with_control( .clone(), database_artifact_repository, ) + // Keyed on the configured flag, as the startup floor check is. .with_disk_policy( tinycloud_config .storage .duckdb - .disk_policy(artifact_collector.is_some(), disk_budget), + .disk_policy(tinycloud_config.storage.sql.chunked_artifacts, disk_budget), ); #[cfg(feature = "duckdb")] duckdb_service.sweep_stale_cache_files().await;