diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a25f1b9..d01c285c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - `POST /delegate` and `POST /revoke` check the signature (and, for a renewal lease, the signature of the stored original it renews) before taking any lock. A forged or slow-to-verify request no longer holds the delegation's chain guards, its spaces' write order or a database transaction while it is checked; it is refused at once. A renewal lease whose original is not stored is also refused before any lock. Inside the transaction only the signed time window is checked again. A resubmitted delegation that is already stored is now also signature-checked before it is answered as existing. Details: TC-2986. - Add opt-in PostgreSQL checkpoint parts for SQL persistence, with explicit artifact/account capacities and streamed hydration. A database stored in parts loads in bounded statements and is not held to the single-row read limit (`storage.sql.max_artifact_read_bytes`); a load that keeps the whole database in memory (a DuckDB export) still waits for one of the large-transfer slots. Details: TC-2917. - Policy/v3: minting a session and registering a policy no longer re-verify signatures or re-run the claim, presentation, credential admission and allow evaluations while holding their spaces' write order and row locks. Under the locks they check that the challenge, registration and roots are unchanged; those decisions were made before the locks, at the same time, from the same inputs. Refusal codes are unchanged, except that `credential-evaluation-changed` and `policy-evaluation-changed` can no longer occur. Details: TC-2986. +- DuckDB saves a complete checkpoint when a write changes its main file, and uses incremental WAL persistence only when that file still matches the stored checkpoint, so acknowledged writes remain recoverable after restart. Details: TC-2990. - SQLite stamp replay deadlines now use the same timestamp encoding as cleanup (TC-2920). - PostgreSQL database artifacts use uncompressed external storage for new writes to avoid repeated decompression during chunked reads (TC-2917). Existing large artifacts require an operator re-toast; verify both storage settings after rollout and follow `scripts/tc2917-README.md` if lock contention deferred the migration. - Encryption: v2 column format with key ids, per-purpose keys and recorded format metadata. Optional strict reads are off by default. Details: TC-2516, TC-2517. diff --git a/tinycloud-core/src/duckdb/database.rs b/tinycloud-core/src/duckdb/database.rs index 824869b8..48def1cf 100644 --- a/tinycloud-core/src/duckdb/database.rs +++ b/tinycloud-core/src/duckdb/database.rs @@ -4,6 +4,7 @@ //! result formatting, native connection open/close and filesystem work do not. use std::collections::BTreeSet; +use std::io::Read; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -12,6 +13,7 @@ use dashmap::DashMap; use sqlparser::ast::Statement; use tokio::sync::{mpsc, oneshot}; +use crate::hash::Blake3Hasher; use crate::request_telemetry::{self, QueryTimer, RequestContext}; use super::{ @@ -38,6 +40,11 @@ enum DbMessage { Wal { response_tx: oneshot::Sender>, DuckDbError>>, }, + WalForCheckpoint { + checkpoint_size: u64, + checkpoint_hash: String, + response_tx: oneshot::Sender>, DuckDbError>>, + }, Shutdown { response_tx: oneshot::Sender<()>, }, @@ -107,6 +114,30 @@ impl DatabaseHandle { .map_err(|_| DuckDbError::Internal("Database actor dropped response".to_string()))? } + /// Copy a WAL only when its local main file still matches the durable + /// checkpoint. Both operations run under the same actor fence. + pub async fn wal_for_checkpoint( + &self, + checkpoint_size: u64, + checkpoint_hash: String, + ) -> Result>, DuckDbError> { + let (response_tx, response_rx) = oneshot::channel(); + self.tx + .send(( + request_telemetry::current(), + DbMessage::WalForCheckpoint { + checkpoint_size, + checkpoint_hash, + response_tx, + }, + )) + .await + .map_err(|_| DuckDbError::Internal("Database actor not available".into()))?; + response_rx + .await + .map_err(|_| DuckDbError::Internal("Database actor dropped response".into()))? + } + /// Stop the actor and close its database before the cache is discarded. pub(crate) async fn shutdown(&self) -> Result<(), DuckDbError> { let (response_tx, response_rx) = oneshot::channel(); @@ -175,7 +206,8 @@ pub fn spawn_actor( e )))); } - DbMessage::Wal { response_tx } => { + DbMessage::Wal { response_tx } + | DbMessage::WalForCheckpoint { response_tx, .. } => { let _ = response_tx.send(Err(DuckDbError::Internal(format!( "Failed to open: {}", e @@ -257,6 +289,19 @@ pub fn spawn_actor( let result = handle_wal(&mode, &file_path); let _ = response_tx.send(result); } + DbMessage::WalForCheckpoint { + checkpoint_size, + checkpoint_hash, + response_tx, + } => { + let result = handle_wal_for_checkpoint( + &mode, + &file_path, + checkpoint_size, + &checkpoint_hash, + ); + let _ = response_tx.send(result); + } DbMessage::Shutdown { response_tx } => { shutdown = Some(response_tx); break; @@ -284,6 +329,44 @@ pub fn spawn_actor( DatabaseHandle { id, tx } } +fn handle_wal_for_checkpoint( + mode: &StorageMode, + file_path: &Path, + checkpoint_size: u64, + checkpoint_hash: &str, +) -> Result>, DuckDbError> { + if !matches!(mode, StorageMode::File(_)) { + return Ok(None); + } + let mut file = + std::fs::File::open(file_path).map_err(|e| DuckDbError::Internal(e.to_string()))?; + if file + .metadata() + .map_err(|e| DuckDbError::Internal(e.to_string()))? + .len() + != checkpoint_size + { + return Ok(None); + } + // DuckDB can place new blocks in the main file while recording only their + // references in the WAL. Such a WAL cannot replay on the older checkpoint. + let mut hasher = Blake3Hasher::new(); + let mut buffer = [0u8; 64 * 1024]; + loop { + let read = file + .read(&mut buffer) + .map_err(|e| DuckDbError::Internal(e.to_string()))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + if hasher.finalize().to_cid(0x55).to_string() != checkpoint_hash { + return Ok(None); + } + handle_wal(mode, file_path) +} + fn handle_wal(mode: &StorageMode, file_path: &Path) -> Result>, DuckDbError> { if !matches!(mode, StorageMode::File(_)) { return Ok(None); diff --git a/tinycloud-core/src/duckdb/service.rs b/tinycloud-core/src/duckdb/service.rs index 909892e3..af59a4b6 100644 --- a/tinycloud-core/src/duckdb/service.rs +++ b/tinycloud-core/src/duckdb/service.rs @@ -522,11 +522,38 @@ impl DuckDbService { let key = (space.to_string(), db_name.to_string()); let expected = self.expectation(&key); - if let Some(wal) = handle - .wal() - .await? - .filter(|wal| wal.len() < MAX_WAL_DELTA_BYTES) - { + let (wal, expected) = if expected != ArtifactExpectation::Absent { + match self + .artifact_repository + .load_meta("duckdb", &key.0, db_name) + .await + .map_err(artifact_error_to_duckdb)? + { + Some(meta) if meta.checkpoint_size_bytes >= 0 => { + // With no recorded lineage, still attach the WAL only to + // the checkpoint it was just verified against. + let bound = if expected == ArtifactExpectation::Any { + ArtifactExpectation::Derived { + revision: meta.revision, + checkpoint_content_hash: meta.checkpoint_content_hash.clone(), + } + } else { + expected + }; + let wal = handle + .wal_for_checkpoint( + meta.checkpoint_size_bytes as u64, + meta.checkpoint_content_hash, + ) + .await?; + (wal, bound) + } + _ => (None, expected), + } + } else { + (None, expected) + }; + if let Some(wal) = wal.filter(|wal| wal.len() < MAX_WAL_DELTA_BYTES) { match self .artifact_repository .save_delta("duckdb", &space.to_string(), db_name, wal, expected.clone()) @@ -1409,4 +1436,343 @@ mod tests { other => panic!("expected schema response, got {other:?}"), } } + #[tokio::test] + async fn tc2990_large_file_backed_insert_recovers_all_acknowledged_rows() { + let repo = artifact_repository().await; + let cache = TempDir::new().unwrap(); + let space = test_space_id("large-file-write-recovery"); + let key = (space.to_string(), "main".to_string()); + let service = file_service(&cache, repo.clone()); + service + .execute( + &space, + "main", + DuckDbRequest::Execute { + schema: Some(vec!["CREATE TABLE items(id BIGINT, body VARCHAR)".into()]), + sql: "INSERT INTO items VALUES (0, 'seed')".into(), + params: vec![], + }, + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await + .unwrap(); + // Hydration guarantees a file-backed actor, including when the native + // size estimate keeps a newly-created database in memory. + service.discard_local_state(&key).await.unwrap(); + let mut expected_count = 1i64; + let mut expected_sum = 0i64; + for (round, rows) in [(1i64, 5i64), (2, 300_000), (3, 5)] { + service.execute(&space,"main",DuckDbRequest::Execute { + schema:None, + sql:format!("INSERT INTO items SELECT {round}*1000000+i, md5(i::VARCHAR) FROM range({rows}) t(i)"), + params:vec![], + },None,"tinycloud.duckdb/write".into(),false).await.unwrap(); + expected_count += rows; + expected_sum += round * 1_000_000 * rows + rows * (rows - 1) / 2; + let fresh = TempDir::new().unwrap(); + let reader = file_service(&fresh, repo.clone()); + let recovered=reader.execute(&space,"main",DuckDbRequest::Query { + sql:"SELECT count(*),sum(id)::BIGINT,sum(CASE WHEN id=0 OR body=md5((id%1000000)::VARCHAR) THEN 1 ELSE 0 END)::BIGINT FROM items".into(), + params:vec![], + },None,"tinycloud.duckdb/read".into(),false).await; + let _ = reader.discard_local_state(&key).await; + let recovered = match recovered { + Ok(result) => result, + Err(error) => { + let _ = service.discard_local_state(&key).await; + panic!("acknowledged round {round} must recover from a fresh cache: {error:?}"); + } + }; + let DuckDbResponse::Query(query) = recovered.response else { + panic!("query required"); + }; + assert_eq!( + query.rows, + vec![vec![ + DuckDbValue::Integer(expected_count), + DuckDbValue::Integer(expected_sum), + DuckDbValue::Integer(expected_count) + ]] + ); + } + service.discard_local_state(&key).await.unwrap(); + let bytes = service.export(&space, "main").await.unwrap(); + let export_dir = TempDir::new().unwrap(); + let path = export_dir.path().join("recovered.duckdb"); + std::fs::write(&path, bytes).unwrap(); + let conn = storage::open_connection(Some(&path), "128MB").unwrap(); + let count: i64 = conn + .query_row("SELECT count(*) FROM items", [], |row| row.get(0)) + .unwrap(); + assert_eq!(count, expected_count); + } + #[tokio::test] + async fn tc2990_wal_requires_both_checkpoint_length_and_cid() { + let repo = artifact_repository().await; + let source_cache = TempDir::new().unwrap(); + let cache = TempDir::new().unwrap(); + let space = test_space_id("wal-checkpoint-identity"); + let key = (space.to_string(), "main".to_string()); + let source = file_service(&source_cache, repo.clone()); + source + .execute( + &space, + "main", + DuckDbRequest::Execute { + schema: Some(vec!["CREATE TABLE items(id INTEGER)".into()]), + sql: "INSERT INTO items VALUES (1)".into(), + params: vec![], + }, + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await + .unwrap(); + source.discard_local_state(&key).await.unwrap(); + let service = file_service(&cache, repo.clone()); + service + .execute( + &space, + "main", + DuckDbRequest::Execute { + schema: None, + sql: "INSERT INTO items VALUES (2)".into(), + params: vec![], + }, + None, + "tinycloud.duckdb/write".into(), + false, + ) + .await + .unwrap(); + let meta = repo + .load_meta("duckdb", &space.to_string(), "main") + .await + .unwrap() + .unwrap(); + assert!( + meta.has_delta(), + "a small write must retain the WAL fast path" + ); + let handle = service.databases.get(&key).unwrap().clone(); + let wal = handle.wal().await.unwrap().unwrap(); + assert!(handle + .wal_for_checkpoint( + meta.checkpoint_size_bytes as u64 + 1, + meta.checkpoint_content_hash.clone() + ) + .await + .unwrap() + .is_none()); + let wrong = crate::hash::hash(b"different checkpoint") + .to_cid(0x55) + .to_string(); + assert!(handle + .wal_for_checkpoint(meta.checkpoint_size_bytes as u64, wrong) + .await + .unwrap() + .is_none()); + let verified = handle + .wal_for_checkpoint( + meta.checkpoint_size_bytes as u64, + meta.checkpoint_content_hash, + ) + .await + .unwrap() + .unwrap(); + assert_eq!( + verified, wal, + "failed identity checks must not checkpoint or truncate the WAL" + ); + service.discard_local_state(&key).await.unwrap(); + } + + /// Repeated small writes to a file-backed database keep riding on the + /// same durable checkpoint: each one saves a WAL, none a full checkpoint. + #[tokio::test] + async fn duckdb_repeated_small_writes_stay_on_the_wal_path() { + let repo = artifact_repository().await; + let source_cache = TempDir::new().unwrap(); + let file_cache = TempDir::new().unwrap(); + let space = test_space_id("duckdb-wal-repeat"); + async fn write(service: &DuckDbService, space: &SpaceId, sql: &str) { + service + .execute( + space, + "analytics", + DuckDbRequest::Execute { + schema: None, + sql: sql.to_string(), + params: Vec::new(), + }, + None, + "tinycloud.duckdb/write".to_string(), + false, + ) + .await + .unwrap(); + } + let source = service(&source_cache, repo.clone()); + write( + &source, + &space, + "CREATE TABLE events (id INTEGER, name VARCHAR)", + ) + .await; + write( + &source, + &space, + "INSERT INTO events VALUES (0, repeat('x', 1000000))", + ) + .await; + let base = repo + .load_meta("duckdb", &space.to_string(), "analytics") + .await + .unwrap() + .unwrap(); + + let service = file_service(&file_cache, repo.clone()); + for (n, sql) in [ + "INSERT INTO events VALUES (1, 'one')", + "INSERT INTO events VALUES (2, 'two')", + "INSERT INTO events VALUES (3, 'three')", + ] + .into_iter() + .enumerate() + { + write(&service, &space, sql).await; + let artifact = repo + .read_all("duckdb", &space.to_string(), "analytics") + .await + .unwrap() + .unwrap(); + assert_eq!(artifact.storage_mode, "checkpoint+wal", "write {n}"); + assert_eq!( + artifact.checkpoint_content_hash, base.checkpoint_content_hash, + "write {n} took a full checkpoint" + ); + } + service + .discard_local_state(&(space.to_string(), "analytics".to_string())) + .await + .unwrap(); + } + + /// A write that reuses freed blocks can leave the main file at the same + /// length while changing its contents. It must not ride on the old + /// checkpoint: every step recovers in a fresh cache with the exact rows. + #[tokio::test] + async fn duckdb_same_length_rewrite_of_freed_blocks_recovers_exact_rows() { + let repo = artifact_repository().await; + let cache = TempDir::new().unwrap(); + let space = test_space_id("duckdb-reuse-same-length"); + let key = (space.to_string(), "main".to_string()); + let service = file_service(&cache, repo.clone()); + let run = |schema: Option>, sql: String| { + let service = &service; + let space = &space; + async move { + service + .execute( + space, + "main", + DuckDbRequest::Execute { + schema, + sql, + params: vec![], + }, + None, + "tinycloud.duckdb/admin".into(), + false, + ) + .await + .unwrap(); + } + }; + run( + Some(vec![ + "CREATE TABLE t1(id BIGINT, body VARCHAR)".into(), + "CREATE TABLE t2(id BIGINT, body VARCHAR)".into(), + ]), + "INSERT INTO t2 VALUES (0, 'seed')".into(), + ) + .await; + service.discard_local_state(&key).await.unwrap(); + let path = service.cache_path(&space, "main"); + let mut expected = 1i64; + let mut same_length_steps = 0; + let steps: Vec<(Option>, &str, i64)> = vec![ + ( + None, + "INSERT INTO t1 SELECT i, md5(i::VARCHAR) FROM range(300000) t(i)", + 0, + ), + ( + Some(vec!["DROP TABLE t1".into()]), + "INSERT INTO t2 SELECT 10000000+i, md5(i::VARCHAR) FROM range(300000) t(i)", + 300000, + ), + ( + None, + "INSERT INTO t2 SELECT 20000000+i, md5(i::VARCHAR) FROM range(250000) t(i)", + 250000, + ), + ( + None, + "INSERT INTO t2 SELECT 30000000+i, md5(i::VARCHAR) FROM range(250000) t(i)", + 250000, + ), + ]; + for (n, (schema, sql, rows)) in steps.into_iter().enumerate() { + let length_before = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0); + run(schema, sql.into()).await; + expected += rows; + if std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0) == length_before { + same_length_steps += 1; + } + // A fresh cache hydrates from the durable artifact alone. The + // check reads every row's content; count(*) alone is answered + // from metadata. + let fresh = TempDir::new().unwrap(); + let reader = file_service(&fresh, repo.clone()); + let got = reader + .execute( + &space, + "main", + DuckDbRequest::Query { + sql: "SELECT count(*), sum(CASE WHEN id=0 OR body=md5((id%10000000)::VARCHAR) \ + THEN 1 ELSE 0 END)::BIGINT FROM t2" + .into(), + params: vec![], + }, + None, + "tinycloud.duckdb/read".into(), + false, + ) + .await; + let _ = reader.discard_local_state(&key).await; + let DuckDbResponse::Query(rows) = got + .unwrap_or_else(|error| panic!("step {n} not recoverable: {error:?}")) + .response + else { + panic!("step {n}: not a query response"); + }; + assert_eq!( + rows.rows, + vec![vec![ + DuckDbValue::Integer(expected), + DuckDbValue::Integer(expected) + ]], + "step {n}" + ); + } + assert!( + same_length_steps > 0, + "no step reused freed blocks at the same file length; the case is not exercised" + ); + service.discard_local_state(&key).await.unwrap(); + } }