From 7ef7cbdd9200b1b5f172438ea62f63509e733da2 Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:26:16 +0000 Subject: [PATCH 1/2] fix(TC-2917): DuckDB cache cleanup without blocking other databases Evicted actors rename their cache files aside under the registration and delete them after releasing it; leftover spill directories no longer block hydration or discard; failed imports leave no staged or imported file; shutting down an exited actor succeeds. --- CHANGELOG.md | 1 + docs/operations/chunked-sql-artifacts.md | 12 +- tinycloud-core/src/database_artifacts.rs | 19 +- tinycloud-core/src/duckdb/database.rs | 234 +++++++++-- tinycloud-core/src/duckdb/service.rs | 489 +++++++++++++++++++---- 5 files changed, 635 insertions(+), 120 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26592e1a..cc10bfdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Add opt-in PostgreSQL checkpoint parts for SQL persistence, with explicit artifact/account capacities and streamed hydration. A database stored in parts loads in bounded statements and is not held to the single-row read limit (`storage.sql.max_artifact_read_bytes`); a load that keeps the whole database in memory (a DuckDB export) still waits for one of the large-transfer slots. Details: TC-2917. - Policy/v3: minting a session and registering a policy no longer re-verify signatures or re-run the claim, presentation, credential admission and allow evaluations while holding their spaces' write order and row locks. Under the locks they check that the challenge, registration and roots are unchanged; those decisions were made before the locks, at the same time, from the same inputs. Refusal codes are unchanged, except that `credential-evaluation-changed` and `policy-evaluation-changed` can no longer occur. Details: TC-2986. - Artifact-part writers prioritize their own predecessor reclamation, keep replacement contention retryable, and distinguish capacity refusals from space-growth limits; existing database layouts can checkpoint before adopting WAL parts. Details: TC-2917. +- DuckDB cache cleanup no longer holds up lookups of other databases while files are deleted; a spill directory left by a stopped process no longer blocks hydration or discard, and a failed import leaves no staged or imported file behind. Details: TC-2917. - DuckDB saves a complete checkpoint when a write changes its main file, and uses incremental WAL persistence only when that file still matches the stored checkpoint, so acknowledged writes remain recoverable after restart. Details: TC-2990. - Opt-in SQL artifact parts also stream cumulative WAL updates, retaining the checkpoint until a full checkpoint is needed. Details: TC-2917. - DuckDB can persist checkpoints, cumulative WAL and imports through the same opt-in bounded artifact parts. Details: TC-2917. diff --git a/docs/operations/chunked-sql-artifacts.md b/docs/operations/chunked-sql-artifacts.md index 8778a22a..e2f91eb7 100644 --- a/docs/operations/chunked-sql-artifacts.md +++ b/docs/operations/chunked-sql-artifacts.md @@ -128,7 +128,12 @@ New DuckDB databases with parts enabled start file-backed under the existing node data directory, regardless of the in-memory threshold. Their actor fence protects checkpoint/WAL copies. Closing or idle-evicting an actor removes its cache files after the native connection is closed; the next access rehydrates -from durable parts. Legacy actors retain their existing storage behavior. +from durable parts. The files are renamed aside while the actor's registration +is held and deleted after it is released, so other databases' lookups do not +wait on the deletion; if the process stops in between, the next hydration of +that database removes them. A spill directory left by a stopped process does +not block hydration, and an import that fails leaves no staged or imported +file. Legacy actors retain their existing storage behavior. For disk headroom, each successfully persisted open database retains at most the configured artifact ceiling in checkpoint plus cumulative WAL. A full @@ -152,7 +157,6 @@ the local main file's length and CID with the stored checkpoint under the actor fence. A mismatch requires a complete checkpoint, including blocks written directly to the main file by a large insert. -The feature remains off. Additional file-removal lock duration, spill-directory -cleanup, import failure cleanup and disk/spill admission conditions still need -their separate acceptance before activation, alongside conversion and the +The feature remains off. DuckDB disk and spill admission still needs its +separate acceptance before activation, alongside conversion and the ceiling-load benchmark. diff --git a/tinycloud-core/src/database_artifacts.rs b/tinycloud-core/src/database_artifacts.rs index c44f8c88..8a48225d 100644 --- a/tinycloud-core/src/database_artifacts.rs +++ b/tinycloud-core/src/database_artifacts.rs @@ -472,8 +472,10 @@ impl Drop for StagingRelease { /// Remove the `.tmp..` files a crashed hydration left beside /// each of `finals`: exactly this sink's temp-name pattern, and never a file -/// this process still has in flight. Run before a hydration begins, under the -/// database's hydration exclusion. +/// this process still has in flight. A directory with that name is a DuckDB +/// spill directory an evicted actor moved aside, and is removed with its +/// contents. Run before a hydration begins, under the database's hydration +/// exclusion. pub(crate) async fn sweep_stale_staging(finals: &[&std::path::Path]) { for path in finals { let (Some(dir), Some(name)) = (path.parent(), path.file_name()) else { @@ -505,8 +507,17 @@ pub(crate) async fn sweep_stale_staging(finals: &[&std::path::Path]) { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .contains(&temp); - if is_staging && !live && tokio::fs::remove_file(&temp).await.is_ok() { - tracing::info!("Removed a staging file left by an interrupted hydration"); + if !is_staging || live { + continue; + } + let removed = match entry.file_type().await { + Ok(kind) if kind.is_dir() => tokio::fs::remove_dir_all(&temp).await, + _ => tokio::fs::remove_file(&temp).await, + }; + if removed.is_ok() { + tracing::info!( + "Removed a staging file left by an interrupted hydration or cleanup" + ); } } } diff --git a/tinycloud-core/src/duckdb/database.rs b/tinycloud-core/src/duckdb/database.rs index b07a7e62..8c168b44 100644 --- a/tinycloud-core/src/duckdb/database.rs +++ b/tinycloud-core/src/duckdb/database.rs @@ -188,18 +188,25 @@ impl DatabaseHandle { } /// Stop the actor and close its database before the cache is discarded. + /// + /// An actor that has already exited (idle eviction, at the same moment as + /// this call) is done too: it drops its receiver only after closing its + /// database and running its cleanup. pub(crate) async fn shutdown(&self) -> Result<(), DuckDbError> { let (response_tx, response_rx) = oneshot::channel(); - self.tx + if self + .tx .send(( request_telemetry::current(), DbMessage::Shutdown { response_tx }, )) .await - .map_err(|_| DuckDbError::Internal("Database actor not available".to_string()))?; - response_rx.await.map_err(|_| { - DuckDbError::Internal("Database actor dropped shutdown response".to_string()) - }) + .is_ok() + { + // A dropped sender means the actor exited with this message queued. + let _ = response_rx.await; + } + Ok(()) } } @@ -431,9 +438,13 @@ pub(crate) fn spawn_actor_with_parts( DatabaseHandle { id, tx } } -/// Keep registration until cleanup is complete. A replacement actor's cache -/// must never be removed by its predecessor. Explicit shutdown already holds -/// the service's per-database operation fence, even if it removed the entry. +/// Keep registration until the cache paths are free. A replacement actor's +/// cache must never be removed by its predecessor. Explicit shutdown already +/// holds the service's per-database operation fence, even if it removed the +/// entry. +/// +/// The files are only renamed aside while the map entry is held, so a lookup +/// of any other database in the same map shard never waits on file deletion. fn deregister_actor( databases: &DashMap<(String, String), DatabaseHandle>, key: (String, String), @@ -446,32 +457,62 @@ fn deregister_actor( if matches!(&entry, Entry::Occupied(entry) if entry.get().id != id) { return; } + let mut failed = false; + let mut aside = Vec::new(); if remove_files { - let mut failed = false; - for candidate in [ - path.to_path_buf(), - PathBuf::from(format!("{}.wal", path.display())), - PathBuf::from(format!("{}.tmp", path.display())), - ] { - let result = if candidate.is_dir() { - std::fs::remove_dir_all(&candidate) - } else { - std::fs::remove_file(&candidate) - }; - if let Err(error) = result { - failed |= error.kind() != std::io::ErrorKind::NotFound; + for candidate in cache_file_paths(path) { + // The sweep before each hydration removes this name if the + // process stops before the deletion below. + let temp = crate::database_artifacts::staging_path(path); + match std::fs::rename(&candidate, &temp) { + Ok(()) => aside.push(temp), + Err(error) => { + failed |= error.kind() != std::io::ErrorKind::NotFound; + crate::database_artifacts::release_staging(&temp); + } } } - if failed { - // Operators use this value-free event when checking cache-disk headroom. - tracing::warn!( - event = "duckdb_cache_cleanup_failed", - "Database cache cleanup failed" - ); + } + match entry { + Entry::Occupied(entry) => { + entry.remove(); } + Entry::Vacant(entry) => drop(entry), + } + for temp in aside { + failed |= remove_cache_entry(&temp).is_err(); + crate::database_artifacts::release_staging(&temp); } - if let Entry::Occupied(entry) = entry { - entry.remove(); + if failed { + // Operators use this value-free event when checking cache-disk headroom. + tracing::warn!( + event = "duckdb_cache_cleanup_failed", + "Database cache cleanup failed" + ); + } +} + +/// The paths DuckDB keeps for the database at `path`: the main file, its WAL +/// and its spill directory. +pub(crate) fn cache_file_paths(path: &Path) -> [PathBuf; 3] { + [ + path.to_path_buf(), + PathBuf::from(format!("{}.wal", path.display())), + PathBuf::from(format!("{}.tmp", path.display())), + ] +} + +/// Remove one cache file, or DuckDB's spill directory with its contents. A +/// path that is already gone counts as removed. +pub(crate) fn remove_cache_entry(path: &Path) -> std::io::Result<()> { + let removed = match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => std::fs::remove_dir_all(path), + Ok(_) => std::fs::remove_file(path), + Err(error) => Err(error), + }; + match removed { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + other => other, } } @@ -1346,4 +1387,139 @@ mod tests { )); assert_eq!(row_count(&conn, "events"), 1); } + + fn schema_write(sql: &str, schema: &str) -> DuckDbRequest { + DuckDbRequest::Execute { + sql: sql.to_string(), + params: Vec::new(), + schema: Some(vec![schema.to_string()]), + } + } + + fn placeholder_handle(id: u64) -> DatabaseHandle { + let (tx, _rx) = mpsc::channel(1); + DatabaseHandle { id, tx } + } + + fn file_actor( + base: &Path, + key: &(String, String), + idle_timeout_secs: u64, + databases: &Arc>, + ) -> DatabaseHandle { + spawn_actor_with_parts( + key.0.clone(), + key.1.clone(), + base.to_string_lossy().into_owned(), + u64::MAX, + true, + idle_timeout_secs, + "128MB".into(), + databases.clone(), + ) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn chunked_actor_close_spares_a_replacement_and_its_files() { + let base = tempfile::TempDir::new().unwrap(); + let databases = Arc::new(DashMap::new()); + let key = ("space-a".to_string(), "main".to_string()); + let actor = file_actor(base.path(), &key, 300, &databases); + databases.insert(key.clone(), actor.clone()); + actor + .execute( + schema_write("INSERT INTO t VALUES (1)", "CREATE TABLE t(i INTEGER)"), + None, + "tinycloud.duckdb/admin".into(), + false, + false, + ) + .await + .unwrap(); + let path = base.path().join(&key.0).join("main.duckdb"); + assert!(path.exists()); + + // A replacement registered under the key; its files are at the same path. + databases.insert(key.clone(), placeholder_handle(u64::MAX)); + actor.shutdown().await.unwrap(); + assert!( + path.exists(), + "the predecessor removed the replacement's cache" + ); + assert_eq!(databases.get(&key).map(|handle| handle.id), Some(u64::MAX)); + + // With the entry vacant (explicit shutdown under the operation fence) + // every cache path goes, the spill directory included. + databases.remove(&key); + let [main, wal, spill] = cache_file_paths(&path); + std::fs::create_dir_all(&spill).unwrap(); + std::fs::write(spill.join("spill"), b"x").unwrap(); + std::fs::write(&wal, b"w").unwrap(); + deregister_actor(&databases, key.clone(), 7, &path, true); + assert!(!main.exists() && !wal.exists() && !spill.exists()); + assert!(std::fs::read_dir(path.parent().unwrap()) + .unwrap() + .next() + .is_none()); + + // Actors that do not own their files never delete them. + std::fs::write(&path, b"keep").unwrap(); + deregister_actor(&databases, key, 7, &path, false); + assert!(path.exists()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn chunked_idle_eviction_frees_cache_paths_before_deregistering() { + let base = tempfile::TempDir::new().unwrap(); + let databases = Arc::new(DashMap::new()); + let key = ("space-evict".to_string(), "main".to_string()); + let path = base.path().join(&key.0).join("main.duckdb"); + let [main, wal, spill] = cache_file_paths(&path); + // A spill directory a crashed query left behind. + std::fs::create_dir_all(&spill).unwrap(); + for i in 0..2_000 { + std::fs::write(spill.join(format!("s{i}")), b"x").unwrap(); + } + let actor = file_actor(base.path(), &key, 1, &databases); + databases.insert(key.clone(), actor.clone()); + actor + .execute( + schema_write("INSERT INTO t VALUES (1)", "CREATE TABLE t(i INTEGER)"), + None, + "tinycloud.duckdb/admin".into(), + false, + false, + ) + .await + .unwrap(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while databases.contains_key(&key) { + assert!( + std::time::Instant::now() < deadline, + "actor must idle-evict" + ); + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + assert!( + !main.exists() && !wal.exists() && !spill.exists(), + "the registration vanished while cache paths remained" + ); + // The renamed-aside files are deleted after the entry is released. + let dir = path.parent().unwrap(); + while std::fs::read_dir(dir).unwrap().next().is_some() { + assert!(std::time::Instant::now() < deadline, "cleanup must finish"); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + + #[tokio::test] + async fn shutdown_of_an_exited_actor_succeeds() { + let base = tempfile::TempDir::new().unwrap(); + let databases = Arc::new(DashMap::new()); + let key = ("space-exit".to_string(), "main".to_string()); + let actor = file_actor(base.path(), &key, 300, &databases); + actor.shutdown().await.unwrap(); + actor.shutdown().await.unwrap(); + } } diff --git a/tinycloud-core/src/duckdb/service.rs b/tinycloud-core/src/duckdb/service.rs index a9f44122..1bce269a 100644 --- a/tinycloud-core/src/duckdb/service.rs +++ b/tinycloud-core/src/duckdb/service.rs @@ -16,7 +16,7 @@ use crate::request_telemetry::{self, QueryTimer}; use super::{ caveats::DuckDbCaveats, - database::{spawn_actor_with_parts, DatabaseHandle}, + database::{cache_file_paths, remove_cache_entry, spawn_actor_with_parts, DatabaseHandle}, storage, types::*, }; @@ -282,86 +282,98 @@ impl DuckDbService { let temp_path = staging_path(&final_path); let _live = StagingRelease(temp_path.clone()); - // Write to temp file first - tokio::fs::write(&temp_path, data) + // Stage, validate and move the file into place. Any failure before the + // rename leaves the staged bytes behind, so it removes them. + let key = (space.to_string(), db_name.to_string()); + let staged = async { + tokio::fs::write(&temp_path, data) + .await + .map_err(|e| DuckDbError::ImportError(e.to_string()))?; + + // Validate with the same locked native policy as live and export connections. + let temp_path_clone = temp_path.clone(); + let max_memory = self.max_memory_per_connection.clone(); + let context = request_telemetry::current(); + tokio::task::spawn_blocking(move || -> Result<(), DuckDbError> { + let _context = request_telemetry::enter(context); + let conn = + storage::open_connection(Some(&temp_path_clone), &max_memory).map_err(|e| { + match e { + DuckDbError::Internal(message) => DuckDbError::ImportError(format!( + "Invalid DuckDB file: {}", + message + )), + other => other, + } + })?; + { + let _timer = QueryTimer::new(); + conn.execute_batch("SELECT 1").map_err(|e| { + DuckDbError::ImportError(format!("Database validation failed: {}", e)) + })?; + } + Ok(()) + }) .await - .map_err(|e| DuckDbError::ImportError(e.to_string()))?; + .map_err(|e| DuckDbError::Internal(format!("Validation task failed: {}", e)))??; - // Validate with the same locked native policy as live and export connections. - let temp_path_clone = temp_path.clone(); - let max_memory = self.max_memory_per_connection.clone(); - let context = request_telemetry::current(); - let valid = tokio::task::spawn_blocking(move || -> Result<(), DuckDbError> { - let _context = request_telemetry::enter(context); - let conn = storage::open_connection(Some(&temp_path_clone), &max_memory).map_err( - |e| match e { - DuckDbError::Internal(message) => { - DuckDbError::ImportError(format!("Invalid DuckDB file: {}", message)) - } - other => other, - }, - )?; - { - let _timer = QueryTimer::new(); - conn.execute_batch("SELECT 1").map_err(|e| { - DuckDbError::ImportError(format!("Database validation failed: {}", e)) - })?; + // Close the previous actor before replacing its files: connection + // shutdown may checkpoint its old WAL into that path. + if let Some((_, handle)) = self.databases.remove(&key) { + handle.shutdown().await?; } - Ok(()) - }) - .await - .map_err(|e| DuckDbError::Internal(format!("Validation task failed: {}", e)))?; - - if let Err(e) = valid { - // Clean up temp file on validation failure + self.lineage.remove(&key); + tokio::fs::rename(&temp_path, &final_path) + .await + .map_err(|e| DuckDbError::ImportError(format!("Failed to finalize import: {}", e))) + } + .await; + if let Err(e) = staged { let _ = tokio::fs::remove_file(&temp_path).await; return Err(e); } - // Close the previous actor before replacing its files: connection - // shutdown may checkpoint its old WAL into that path. - let key = (space.to_string(), db_name.to_string()); - if let Some((_, handle)) = self.databases.remove(&key) { - handle.shutdown().await?; + // From here the imported file is in the cache; a failure to persist it + // must not leave it there to be served. + let persisted = async { + if self + .artifact_repository + .chunked_checkpoints("duckdb", &key.0, db_name) + { + let snapshot = tempfile::NamedTempFile::new() + .map_err(|e| DuckDbError::Internal(e.to_string()))?; + tokio::fs::write(snapshot.path(), data) + .await + .map_err(|e| DuckDbError::ImportError(e.to_string()))?; + self.artifact_repository + .save_snapshot( + "duckdb", + &key.0, + db_name, + snapshot, + ArtifactExpectation::Any, + ) + .await + .map(|_| ()) + .map_err(artifact_error_to_duckdb) + } else { + self.artifact_repository + .save( + "duckdb", + &key.0, + db_name, + data.to_vec(), + ArtifactExpectation::Any, + ) + .await + .map(|_| ()) + .map_err(artifact_error_to_duckdb) + } } - self.lineage.remove(&key); - tokio::fs::rename(&temp_path, &final_path) - .await - .map_err(|e| DuckDbError::ImportError(format!("Failed to finalize import: {}", e)))?; - let persisted = if self - .artifact_repository - .chunked_checkpoints("duckdb", &key.0, db_name) - { - let snapshot = - tempfile::NamedTempFile::new().map_err(|e| DuckDbError::Internal(e.to_string()))?; - tokio::fs::write(snapshot.path(), data) - .await - .map_err(|e| DuckDbError::ImportError(e.to_string()))?; - self.artifact_repository - .save_snapshot( - "duckdb", - &key.0, - db_name, - snapshot, - ArtifactExpectation::Any, - ) - .await - .map(|_| ()) - } else { - self.artifact_repository - .save( - "duckdb", - &key.0, - db_name, - data.to_vec(), - ArtifactExpectation::Any, - ) - .await - .map(|_| ()) - }; + .await; if let Err(error) = persisted { self.discard_local_state(&key).await?; - return Err(artifact_error_to_duckdb(error)); + return Err(error); } Ok(()) @@ -759,18 +771,15 @@ fn validate_payload( } async fn remove_duckdb_cache_files(path: &Path) -> Result<(), DuckDbError> { - for candidate in [ - path.to_path_buf(), - PathBuf::from(format!("{}.tmp", path.display())), - PathBuf::from(format!("{}.wal", path.display())), - ] { - match tokio::fs::remove_file(&candidate).await { - Ok(_) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => return Err(DuckDbError::Internal(e.to_string())), - } - } - Ok(()) + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || { + cache_file_paths(&path) + .iter() + .try_for_each(|candidate| remove_cache_entry(candidate)) + }) + .await + .map_err(|e| DuckDbError::Internal(e.to_string()))? + .map_err(|e| DuckDbError::Internal(e.to_string())) } fn duckdb_wal_path(path: &Path) -> PathBuf { @@ -2204,4 +2213,318 @@ mod tests { .unwrap(); assert_eq!(count, expected_count); } + + fn write_request(sql: &str, schema: Option<&str>) -> DuckDbRequest { + DuckDbRequest::Execute { + schema: schema.map(|schema| vec![schema.to_string()]), + sql: sql.into(), + params: vec![], + } + } + + async fn query_rows( + service: &DuckDbService, + space: &SpaceId, + sql: &str, + ) -> Result>, DuckDbError> { + let result = service + .execute( + space, + "main", + DuckDbRequest::Query { + sql: sql.into(), + params: vec![], + }, + None, + "tinycloud.duckdb/read".into(), + false, + ) + .await?; + let DuckDbResponse::Query(query) = result.response else { + panic!("query required"); + }; + Ok(query.rows) + } + + fn dir_entries(dir: &Path) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .map(|entries| { + entries + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default(); + names.sort(); + names + } + + fn chunked_repository( + conn: &sea_orm::DatabaseConnection, + artifact_bytes: u64, + ) -> Arc { + Arc::new( + SeaOrmDatabaseArtifactRepository::new(conn.clone()).with_chunk_limits(Some( + crate::database_artifacts::chunks::ChunkLimits { + artifact_bytes, + account_bytes: 128 << 20, + }, + )), + ) + } + + /// A valid DuckDB file of more than 1 MiB, written under the node's policy. + fn large_duckdb_file() -> Vec { + let scratch = TempDir::new().unwrap(); + let path = scratch.path().join("large.duckdb"); + { + let conn = storage::open_connection(Some(&path), "128MB").unwrap(); + conn.execute_batch( + "CREATE TABLE big AS SELECT i, md5(i::VARCHAR) AS s FROM range(200000) r(i); \ + CHECKPOINT;", + ) + .unwrap(); + } + let bytes = std::fs::read(&path).unwrap(); + assert!(bytes.len() > 1 << 20); + bytes + } + + #[tokio::test] + async fn duckdb_database_names_cannot_leave_the_cache_directory() { + let root = TempDir::new().unwrap(); + let base = root.path().join("cache"); + std::fs::create_dir_all(&base).unwrap(); + let service = DuckDbService::new( + base.to_string_lossy().into_owned(), + 0, + 300, + "128MB".into(), + artifact_repository().await, + ); + let space = test_space_id("traversal"); + for name in ["../../escape", "..", "a/b", "a\\b", "", "x\0y", "../escape"] { + let executed = service + .execute( + &space, + name, + write_request( + "INSERT INTO t VALUES (1)", + Some("CREATE TABLE t(i INTEGER)"), + ), + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await; + assert!( + matches!(executed, Err(DuckDbError::PermissionDenied(_))), + "{name:?}: {executed:?}" + ); + let imported = service.import_db(&space, name, b"x").await; + assert!( + matches!(imported, Err(DuckDbError::PermissionDenied(_))), + "{name:?}: {imported:?}" + ); + } + assert_eq!(dir_entries(root.path()), vec!["cache".to_string()]); + assert!(dir_entries(&base.join(space.to_string())).is_empty()); + } + + #[tokio::test] + async fn duckdb_leftover_spill_directories_do_not_block_the_database() { + let repo = artifact_repository().await; + let cache = TempDir::new().unwrap(); + let service = service(&cache, repo); + let space = test_space_id("spill-leftover"); + let key = (space.to_string(), "main".to_string()); + service + .execute( + &space, + "main", + write_request( + "INSERT INTO t VALUES (1)", + Some("CREATE TABLE t(i INTEGER)"), + ), + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await + .unwrap(); + service.discard_local_state(&key).await.unwrap(); + // What a killed process leaves: DuckDB's own spill directory, and one + // an evicted actor had moved aside for deletion. + let path = service.cache_path(&space, "main"); + let spill = PathBuf::from(format!("{}.tmp", path.display())); + let aside = PathBuf::from(format!("{}.tmp.999999999.0", path.display())); + for dir in [&spill, &aside] { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(dir.join("duckdb_temp_storage-0.tmp"), b"x").unwrap(); + } + assert_eq!( + query_rows(&service, &space, "SELECT count(*) FROM t") + .await + .unwrap(), + vec![vec![DuckDbValue::Integer(1)]] + ); + assert!(!spill.exists() && !aside.exists()); + + std::fs::create_dir_all(&spill).unwrap(); + std::fs::write(spill.join("duckdb_temp_storage-0.tmp"), b"x").unwrap(); + service.discard_local_state(&key).await.unwrap(); + assert!(dir_entries(path.parent().unwrap()).is_empty()); + } + + #[tokio::test] + async fn duckdb_import_that_cannot_be_finalized_leaves_no_staging_file() { + let repo = artifact_repository().await; + let cache = TempDir::new().unwrap(); + let service = service(&cache, repo.clone()); + let space = test_space_id("import-finalize"); + let path = service.cache_path(&space, "main"); + // A non-empty directory at the final path makes the rename fail. + std::fs::create_dir_all(path.join("blocker")).unwrap(); + let imported = service + .import_db(&space, "main", &large_duckdb_file()) + .await; + assert!( + matches!(imported, Err(DuckDbError::ImportError(_))), + "{imported:?}" + ); + assert_eq!(dir_entries(path.parent().unwrap()), vec!["main.duckdb"]); + assert!(repo + .load_meta("duckdb", &space.to_string(), "main") + .await + .unwrap() + .is_none()); + } + + #[tokio::test] + async fn postgres_chunked_duckdb_refused_and_invalid_imports_leave_no_files() { + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_chunked_duckdb_refused_and_invalid_imports_leave_no_files", + ) + .await + else { + return; + }; + let bytes = large_duckdb_file(); + let repo = chunked_repository(&fixture.conn, 1 << 20); + let cache = TempDir::new().unwrap(); + let service = service(&cache, repo.clone()); + let space = test_space_id("chunked-import-refused"); + let dir = cache.path().join(space.to_string()); + + let refused = service.import_db(&space, "main", &bytes).await; + assert!( + matches!(refused, Err(DuckDbError::DatabaseSizeLimit)), + "{refused:?}" + ); + assert!(dir_entries(&dir).is_empty(), "{:?}", dir_entries(&dir)); + assert!(repo + .load_meta("duckdb", &space.to_string(), "main") + .await + .unwrap() + .is_none()); + + let invalid = service + .import_db(&space, "main", b"not a duckdb file") + .await; + assert!( + matches!(invalid, Err(DuckDbError::ImportError(_))), + "{invalid:?}" + ); + assert!(dir_entries(&dir).is_empty(), "{:?}", dir_entries(&dir)); + + assert!(query_rows(&service, &space, "SELECT count(*) FROM big") + .await + .is_err()); + } + + #[tokio::test] + async fn postgres_chunked_duckdb_substituted_part_is_refused_before_open() { + use sea_orm::{ConnectionTrait, DbBackend, Statement}; + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_chunked_duckdb_substituted_part_is_refused_before_open", + ) + .await + else { + return; + }; + let repo = chunked_repository(&fixture.conn, 64 << 20); + let cache = TempDir::new().unwrap(); + let service = service(&cache, repo.clone()); + let victim = test_space_id("victim"); + let other = test_space_id("other"); + for (space, value) in [(&victim, "victim-secret"), (&other, "other-secret")] { + service + .execute( + space, + "main", + write_request( + &format!("INSERT INTO t VALUES ('{value}')"), + Some("CREATE TABLE t(v VARCHAR)"), + ), + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await + .unwrap(); + } + let victim_meta = repo + .load_meta("duckdb", &victim.to_string(), "main") + .await + .unwrap() + .unwrap(); + let other_meta = repo + .load_meta("duckdb", &other.to_string(), "main") + .await + .unwrap() + .unwrap(); + assert_eq!( + victim_meta.checkpoint_size_bytes, + other_meta.checkpoint_size_bytes + ); + let key = (victim.to_string(), "main".to_string()); + service.discard_local_state(&key).await.unwrap(); + + // Storage-level substitution: the victim's chunks get the other + // tenant's bytes of the same length. + fixture + .conn + .execute_unprepared("ALTER TABLE artifact_chunk DISABLE TRIGGER USER") + .await + .unwrap(); + let substituted = fixture + .conn + .execute(Statement::from_sql_and_values( + DbBackend::Postgres, + "UPDATE artifact_chunk v SET payload = o.payload \ + FROM database_artifact dv, database_artifact do_, artifact_chunk o \ + WHERE dv.space = $1 AND dv.name = 'main' AND v.part_id = dv.checkpoint_part_id \ + AND do_.space = $2 AND do_.name = 'main' \ + AND o.part_id = do_.checkpoint_part_id AND o.ordinal = v.ordinal", + [victim.to_string().into(), other.to_string().into()], + )) + .await + .unwrap() + .rows_affected(); + fixture + .conn + .execute_unprepared("ALTER TABLE artifact_chunk ENABLE TRIGGER USER") + .await + .unwrap(); + assert!(substituted >= 1); + + let outcome = query_rows(&service, &victim, "SELECT v FROM t").await; + assert!( + matches!(&outcome, Err(DuckDbError::Internal(message)) if message.contains("content hash")), + "{outcome:?}" + ); + let dir = cache.path().join(victim.to_string()); + assert!(dir_entries(&dir).is_empty(), "{:?}", dir_entries(&dir)); + assert!(!service.databases.contains_key(&key)); + } } From 327b09c04fa5e7fff38d2a1944fc6e583faf2bd3 Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:39:21 +0000 Subject: [PATCH 2/2] fix(TC-2917): carry DuckDB cleanup order in the types Renaming aside needs the held registration, releasing it consumes the entry, and only the released form can be deleted: reordering either step no longer compiles (#601 review C2). --- tinycloud-core/src/duckdb/database.rs | 90 ++++++++++++++++++++------- 1 file changed, 67 insertions(+), 23 deletions(-) diff --git a/tinycloud-core/src/duckdb/database.rs b/tinycloud-core/src/duckdb/database.rs index 8c168b44..b121c0e8 100644 --- a/tinycloud-core/src/duckdb/database.rs +++ b/tinycloud-core/src/duckdb/database.rs @@ -438,6 +438,8 @@ pub(crate) fn spawn_actor_with_parts( DatabaseHandle { id, tx } } +type ActorEntry<'a> = dashmap::mapref::entry::Entry<'a, (String, String), DatabaseHandle>; + /// Keep registration until the cache paths are free. A replacement actor's /// cache must never be removed by its predecessor. Explicit shutdown already /// holds the service's per-database operation fence, even if it removed the @@ -445,6 +447,9 @@ pub(crate) fn spawn_actor_with_parts( /// /// The files are only renamed aside while the map entry is held, so a lookup /// of any other database in the same map shard never waits on file deletion. +/// The order is carried by the types in [`cleanup`]: renaming aside needs the +/// held entry, releasing consumes it, and only the released form can be +/// deleted. fn deregister_actor( databases: &DashMap<(String, String), DatabaseHandle>, key: (String, String), @@ -452,43 +457,82 @@ fn deregister_actor( path: &Path, remove_files: bool, ) { - use dashmap::mapref::entry::Entry; let entry = databases.entry(key); - if matches!(&entry, Entry::Occupied(entry) if entry.get().id != id) { + if matches!(&entry, ActorEntry::Occupied(entry) if entry.get().id != id) { return; } - let mut failed = false; - let mut aside = Vec::new(); - if remove_files { + let moved = if remove_files { + cleanup::move_aside(&entry, path) + } else { + cleanup::MovedAside::default() + }; + if cleanup::delete_aside(cleanup::release(entry, moved)) { + // Operators use this value-free event when checking cache-disk headroom. + tracing::warn!( + event = "duckdb_cache_cleanup_failed", + "Database cache cleanup failed" + ); + } +} + +/// The cleanup steps, in a module of their own so that a [`cleanup::Released`] +/// can only come from [`cleanup::release`]. +mod cleanup { + use std::path::{Path, PathBuf}; + + use super::{cache_file_paths, remove_cache_entry, ActorEntry}; + + /// Cache files renamed to staging names, not yet deleted. + #[derive(Default)] + pub(super) struct MovedAside { + temps: Vec, + failed: bool, + } + + /// [`MovedAside`] once the registration is gone: the only form that can + /// be deleted. + pub(super) struct Released(MovedAside); + + /// Rename the cache files at `path` aside while `_held` keeps the + /// registration, so the paths are free before it can disappear. The + /// sweep before each hydration removes the names if the process stops + /// before [`delete_aside`]. + pub(super) fn move_aside(_held: &ActorEntry<'_>, path: &Path) -> MovedAside { + let mut moved = MovedAside::default(); for candidate in cache_file_paths(path) { - // The sweep before each hydration removes this name if the - // process stops before the deletion below. let temp = crate::database_artifacts::staging_path(path); match std::fs::rename(&candidate, &temp) { - Ok(()) => aside.push(temp), + Ok(()) => moved.temps.push(temp), Err(error) => { - failed |= error.kind() != std::io::ErrorKind::NotFound; + moved.failed |= error.kind() != std::io::ErrorKind::NotFound; crate::database_artifacts::release_staging(&temp); } } } + moved } - match entry { - Entry::Occupied(entry) => { - entry.remove(); + + /// Drop the registration (or the vacant entry's lock), releasing the + /// shard. + pub(super) fn release(entry: ActorEntry<'_>, moved: MovedAside) -> Released { + match entry { + ActorEntry::Occupied(entry) => { + entry.remove(); + } + ActorEntry::Vacant(entry) => drop(entry), } - Entry::Vacant(entry) => drop(entry), - } - for temp in aside { - failed |= remove_cache_entry(&temp).is_err(); - crate::database_artifacts::release_staging(&temp); + Released(moved) } - if failed { - // Operators use this value-free event when checking cache-disk headroom. - tracing::warn!( - event = "duckdb_cache_cleanup_failed", - "Database cache cleanup failed" - ); + + /// Delete what was moved aside, outside the map lock. True if anything + /// failed, including the renames. + pub(super) fn delete_aside(Released(moved): Released) -> bool { + let mut failed = moved.failed; + for temp in moved.temps { + failed |= remove_cache_entry(&temp).is_err(); + crate::database_artifacts::release_staging(&temp); + } + failed } }