From 29470c15b6dd21024f5f8246ac87307371380817 Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:08:55 +0000 Subject: [PATCH 1/5] fix: order legacy write locks and slim the checkpoint transaction (TC-2954) - A protected legacy meeting write takes its space's sequence guard before BEGIN on PostgreSQL too, so it never holds a pooled connection while queueing for it. - A checkpoint save reads its row's metadata before opening the bulk-timeout transaction for the write. - The pool logs a warning when it closes a connection released inside a transaction. - Regression for a deadline during a SeaORM begin through a slow link. The idle-in-transaction timeout moved to its own PR (#586). --- CHANGELOG.md | 4 + tinycloud-core/src/database_artifacts.rs | 24 ++--- tinycloud-core/src/db.rs | 98 +++++++++++++++++++-- tinycloud-core/src/db_pool/release_tests.rs | 92 +++++++++++++++++++ vendor/sqlx-core/PATCH.md | 5 +- vendor/sqlx-core/src/pool/connection.rs | 2 + 6 files changed, 207 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f601623..819dc0d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ ## [Unreleased] ### Changes - PostgreSQL: every pooled connection sets `idle_in_transaction_session_timeout` (`database.idle_in_transaction_timeout_ms`, env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`, default 300000 = 5 min, `0` off), so the server ends a transaction left idle that long and the pool replaces the session. A non-zero value of 240 s or less is refused at startup: a protected legacy meeting write legitimately holds its transaction across an upload of up to 240 s. It is not set in `statement_timeout_mode = "local"` (transaction-pooling proxies). `database.idle_in_transaction_timeout_ms`, `statement_timeout_ms` and `bulk_statement_timeout_ms` above 2147483647 (PostgreSQL's largest) are now refused at startup instead of failing every connection. Details: TC-2954. +- A registered Policy/v3 root never authorizes a request by itself, whether the invocation cites it alone or beside other proofs, and whichever codec or multibase encoding of its CID it uses: policy roots are matched by their canonical identity. `/invoke` and the routes that share its policy check, the encryption network and key routes, and `POST /policy/v3/deliveries/authorize` answer 403. Access under a policy needs a session minted under the policy's conditions. Details: TC-2958. +- `POST /policy/v3/deliveries/authorize` runs the same policy check as `/invoke`: an invocation under a policy session must meet the session's conditions (a single proof, an invocation lifetime of at most 60 seconds, its scope and liveness), and a session authorizes delivery only of a share under its own policy; otherwise the answer is 403 `delivery-authorization-invalid`. Details: TC-2958. +- A protected legacy meeting write now waits for its space's write order before opening its transaction on PostgreSQL too (as on SQLite), so it no longer holds a pooled connection while queueing behind other writers to the same space; those writers now wait for its upload instead. Details: TC-2954. +- A SQL checkpoint save over 8 MiB reads its row's metadata before opening the transaction that uploads it, so that transaction holds no table lock while the payload is sent. A pooled connection closed because it was released inside a transaction is now logged as a warning. Details: TC-2954. - SQLite stamp replay deadlines now use the same timestamp encoding as cleanup (TC-2920). - PostgreSQL database artifacts use uncompressed external storage for new writes to avoid repeated decompression during chunked reads (TC-2917). Existing large artifacts require an operator re-toast; verify both storage settings after rollout and follow `scripts/tc2917-README.md` if lock contention deferred the migration. - Encryption: v2 column format with key ids, per-purpose keys and recorded format metadata. Optional strict reads are off by default. Details: TC-2516, TC-2517. diff --git a/tinycloud-core/src/database_artifacts.rs b/tinycloud-core/src/database_artifacts.rs index 184533ee..8a143563 100644 --- a/tinycloud-core/src/database_artifacts.rs +++ b/tinycloud-core/src/database_artifacts.rs @@ -978,17 +978,10 @@ impl DatabaseArtifactRepository for SeaOrmDatabaseArtifactRepository { .format(&Rfc3339) .expect("current timestamps should format as RFC3339"); - // A checkpoint larger than a WAL delta may take longer to upload than - // the pool's base statement timeout allows, so its write runs under - // the bulk one (TC-1054). Smaller ones, like every delta, fit the - // base timeout and pay no extra round trip. - let scope = if payload.len() > BASE_TIMEOUT_PAYLOAD_BYTES { - self.conn.begin_bulk().await? - } else { - BulkScope::Pool(self.conn.clone()) - }; // For Absent, the primary key on the create below asserts absence - // atomically. Details: TC-2214 + // atomically. Details: TC-2214. The read runs on the pool, outside the + // bulk transaction: the CAS on the write below needs no transaction, + // so no lock or snapshot is held while the payload uploads (TC-2954). let existing = if expected == ArtifactExpectation::Absent { None } else { @@ -1004,7 +997,7 @@ impl DatabaseArtifactRepository for SeaOrmDatabaseArtifactRepository { .column(database_artifact::Column::SizeBytes) .column(database_artifact::Column::CheckpointContentHash) .into_tuple::<(i64, i64, String)>() - .one(&scope) + .one(&self.conn) .await? .map( |(revision, size_bytes, checkpoint_content_hash)| CurrentArtifact { @@ -1047,6 +1040,15 @@ impl DatabaseArtifactRepository for SeaOrmDatabaseArtifactRepository { #[cfg(test)] self.wait_at_race_barrier().await; + // A checkpoint larger than a WAL delta may take longer to upload than + // the pool's base statement timeout allows, so its write runs under + // the bulk one (TC-1054). Smaller ones, like every delta, fit the + // base timeout and pay no extra round trip. + let scope = if payload.len() > BASE_TIMEOUT_PAYLOAD_BYTES { + self.conn.begin_bulk().await? + } else { + BulkScope::Pool(self.conn.clone()) + }; match existing.as_ref() { Some(existing) => { // Compare-and-swap on the (revision, checkpoint) this save read. diff --git a/tinycloud-core/src/db.rs b/tinycloud-core/src/db.rs index 65237b79..aff6c82d 100644 --- a/tinycloud-core/src/db.rs +++ b/tinycloud-core/src/db.rs @@ -3075,7 +3075,13 @@ where // never hold a transaction, pool connection, sequence or writer // lock during upload. let protected = !legacy_spaces.is_empty(); - let _legacy_sqlite_sequence = if protected && self.writer_lock.is_some() { + // Protected writes take the sequence guards before BEGIN on every + // backend (TC-2954). Taken after BEGIN, as PostgreSQL once did, a + // protected writer queued for them while pinning its pooled + // connection, behind ordinary writers that hold them and need a + // connection for their own BEGIN. Same-space ordinary writers now + // wait out the protected upload, holding no connection. + let _legacy_sequence = if protected { self.acquire_space_sequence_guards(&sequence_spaces).await } else { Vec::new() @@ -3102,10 +3108,10 @@ where ) .await?; } - // PG sequence guards are taken after upload. SQLite takes them - // before its writer mutex and BEGIN. The protected SQLite writer - // already holds both guards above. - let _sequence_guards = if protected && self.writer_lock.is_some() { + // Ordinary writes take the sequence guards only now, after upload + // (SQLite: before its writer mutex and BEGIN). A protected writer + // already holds them. + let _sequence_guards = if protected { Vec::new() } else { self.acquire_space_sequence_guards(&sequence_spaces).await @@ -7322,6 +7328,88 @@ mod test { assert_eq!(reader.kv_get(&space, &path).await.unwrap().unwrap().1, hash); } + /// TC-2954: a protected legacy writer queues for its space's sequence + /// guard before BEGIN, so waiting behind a same-space writer pins no + /// pooled connection; that writer needs one for its own BEGIN. + #[tokio::test] + async fn postgres_legacy_writer_queues_for_the_sequence_guard_without_a_connection() { + use crate::storage::memory::MemoryStaging; + use futures::io::AsyncWriteExt; + let Some(schema) = crate::test_support::PostgresTestSchema::new( + "postgres_legacy_writer_queues_for_the_sequence_guard_without_a_connection", + ) + .await + else { + return; + }; + let storage = LegacyPausedStore::default(); + let db = SpaceDatabase::new( + schema.conn.clone(), + storage.clone(), + StaticSecret::new(vec![0; 32]).unwrap(), + ) + .await + .unwrap(); + let space = test_space_id("pg-legacy-guard-before-begin"); + space::ActiveModel { + id: Set(SpaceIdWrap(space.clone())), + } + .insert(&db.conn) + .await + .unwrap(); + let path: Path = "xyz.tinycloud.tinychat/connectors/fireflies/transcript/queued" + .parse() + .unwrap(); + let pool = schema.conn.get_postgres_connection_pool(); + let in_use = || pool.size() as usize - pool.num_idle(); + // Setup's connections go back to the pool on spawned tasks. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while in_use() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("setup returns its connections"); + // A same-space writer holds the guard; the upload itself never pauses. + let guard = db + .acquire_space_sequence_guards(std::slice::from_ref(&space)) + .await; + storage.release.notify_one(); + let writer_db = db.clone(); + let writer_space = space.clone(); + let writer_path = path.clone(); + let writer = tokio::spawn(async move { + let mut stage = HashBuffer::new(Vec::new()); + stage.write_all(b"queued legacy write").await.unwrap(); + writer_db + .invoke_internal_kv_put::( + writer_space, + writer_path, + Metadata(Default::default()), + stage, + None, + ) + .await + .map_err(|error| error.to_string()) + }); + // Long enough for the writer to register, and, were it to begin before + // queueing, to begin and upload too. + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!(!writer.is_finished(), "the writer waits for the guard"); + assert_eq!( + in_use(), + 0, + "a writer queued for the sequence guard holds no pooled connection" + ); + drop(guard); + let hash = tokio::time::timeout(std::time::Duration::from_secs(10), writer) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(db.kv_get(&space, &path).await.unwrap().unwrap().1, hash); + } + #[tokio::test] async fn legacy_freeze_during_failed_upload_refunds_and_rejects_later_write_before_upload() { use crate::storage::{memory::MemoryStaging, StoreSize}; diff --git a/tinycloud-core/src/db_pool/release_tests.rs b/tinycloud-core/src/db_pool/release_tests.rs index 7ee3c9d5..100cf46f 100644 --- a/tinycloud-core/src/db_pool/release_tests.rs +++ b/tinycloud-core/src/db_pool/release_tests.rs @@ -3,7 +3,11 @@ //! that transaction: acknowledged, then lost when it ends without a commit. use sea_orm::sqlx::{self, postgres::PgPoolOptions, Executor, PgPool}; +use sea_orm::TransactionTrait; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; use crate::request_cancel::{run_with_deadline, DeadlineExceeded}; use crate::test_support::PostgresTestSchema; @@ -114,6 +118,94 @@ async fn postgres_a_deadline_during_begin_returns_the_connection_outside_a_trans ); } +/// A TCP proxy to `upstream` that delays each server-to-client chunk by +/// `delay_ms` while it is non-zero, as a slow database link does. Returns its +/// port. +async fn delaying_proxy(upstream: (String, u16), delay_ms: Arc) -> u16 { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((client, _)) = listener.accept().await { + let server = tokio::net::TcpStream::connect((upstream.0.as_str(), upstream.1)) + .await + .unwrap(); + let (mut client_read, mut client_write) = client.into_split(); + let (mut server_read, mut server_write) = server.into_split(); + tokio::spawn(async move { + let _ = tokio::io::copy(&mut client_read, &mut server_write).await; + }); + let delay_ms = Arc::clone(&delay_ms); + tokio::spawn(async move { + let mut buf = vec![0; 64 * 1024]; + while let Ok(n @ 1..) = server_read.read(&mut buf).await { + let delay = delay_ms.load(Ordering::SeqCst); + if delay > 0 { + tokio::time::sleep(Duration::from_millis(delay)).await; + } + if client_write.write_all(&buf[..n]).await.is_err() { + return; + } + } + }); + } + }); + port +} + +/// The node's own path: a SeaORM transaction whose `BEGIN` reply is still on +/// a slow link when the request deadline cuts it. The connection goes back +/// to the pool outside any transaction and the next borrower's write stays. +#[tokio::test] +async fn postgres_a_deadline_during_a_sea_orm_begin_returns_the_connection_outside_a_transaction() { + let Some(fixture) = PostgresTestSchema::new( + "postgres_a_deadline_during_a_sea_orm_begin_returns_the_connection_outside_a_transaction", + ) + .await + else { + return; + }; + let admin = fixture.conn.get_postgres_connection_pool(); + let direct = single_connection_pool(&fixture).await; + let options = direct.connect_options().as_ref().clone(); + direct.close().await; + let delay_ms = Arc::new(AtomicU64::new(0)); + let port = delaying_proxy( + (options.get_host().to_owned(), options.get_port()), + Arc::clone(&delay_ms), + ) + .await; + let pool = PgPoolOptions::new() + .max_connections(1) + .test_before_acquire(false) + .connect_with(options.host("127.0.0.1").port(port)) + .await + .unwrap(); + let pid = backend_pid(&pool).await; + let db = sea_orm::SqlxPostgresConnector::from_sqlx_postgres_pool(pool.clone()); + // Only once the connection is back in the pool, so the deadline cuts the + // BEGIN and not the acquire waiting for the release ping. + while pool.num_idle() == 0 { + tokio::task::yield_now().await; + } + + delay_ms.store(300, Ordering::SeqCst); + let cut = run_with_deadline(Some(Duration::from_millis(100)), async { + db.begin().await.map(drop) + }) + .await; + assert_eq!(cut.err(), Some(DeadlineExceeded)); + delay_ms.store(0, Ordering::SeqCst); + + assert_eq!( + next_borrower_writes(&pool, admin).await, + NextBorrower { + pid, + state: "idle".into(), + write_survived: true, + } + ); +} + /// A connection whose session is still inside a transaction when it is /// released is closed instead of pooled, whatever left that transaction open. #[tokio::test] diff --git a/vendor/sqlx-core/PATCH.md b/vendor/sqlx-core/PATCH.md index 82a23e4f..9a077e3b 100644 --- a/vendor/sqlx-core/PATCH.md +++ b/vendor/sqlx-core/PATCH.md @@ -142,8 +142,9 @@ released with a transaction open. - `src/pool/connection.rs`, `Floating::return_to_pool`: after a successful release ping, a connection whose `in_server_transaction_block()` is true is closed (graceful, bounded by `CLOSE_ON_DROP_TIMEOUT`) instead of released. - Closing the session makes the server roll the transaction back. The pool opens - a replacement on the next acquire. + Closing the session makes the server roll the transaction back, and a + `warn!` ("closing a pooled connection released inside a transaction") + records each such leak. The pool opens a replacement on the next acquire. ### Re-applying on a vendor bump diff --git a/vendor/sqlx-core/src/pool/connection.rs b/vendor/sqlx-core/src/pool/connection.rs index 6b106df9..8daa4217 100644 --- a/vendor/sqlx-core/src/pool/connection.rs +++ b/vendor/sqlx-core/src/pool/connection.rs @@ -346,6 +346,8 @@ impl Floating> { // including any drop-time rollback. A session still inside a transaction would run // the next borrower's statements in it, so close it; ending the session ends the // transaction. Bounded like the close above. + // A leak the drop guards missed: logged so it is seen, not only absorbed. + tracing::warn!("closing a pooled connection released inside a transaction"); let _ = crate::rt::timeout(CLOSE_ON_DROP_TIMEOUT, self.close()).await; false } else { From 2f4b5d2b49ed45f84e5153495d486726f6d23fbf Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:49:42 +0000 Subject: [PATCH 2/5] feat(TC-2961): artifact read limit and a cap on concurrent large transfers A database whose stored checkpoint or WAL delta exceeds storage.sql.max_artifact_read_bytes (default 64 MiB, 0 = off) is not loaded: 507 sql_database_size_limit before any slice is read. At most two artifact transfers over 8 MiB run at once per repository; further ones wait holding no connection. Documents that the checkpoint back-off is in-process. --- CHANGELOG.md | 1 + docs/sql-local-durability.md | 5 +- tinycloud-core/src/database_artifacts.rs | 246 +++++++++++++++++++++++ tinycloud-core/src/sql/service.rs | 72 +++++++ tinycloud-core/src/sql/types.rs | 5 + tinycloud-node-server/src/config.rs | 11 + tinycloud-node-server/src/lib.rs | 22 +- tinycloud-node-server/src/routes/mod.rs | 4 +- 8 files changed, 357 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 819dc0d8..3170614f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - `POST /policy/v3/deliveries/authorize` runs the same policy check as `/invoke`: an invocation under a policy session must meet the session's conditions (a single proof, an invocation lifetime of at most 60 seconds, its scope and liveness), and a session authorizes delivery only of a share under its own policy; otherwise the answer is 403 `delivery-authorization-invalid`. Details: TC-2958. - A protected legacy meeting write now waits for its space's write order before opening its transaction on PostgreSQL too (as on SQLite), so it no longer holds a pooled connection while queueing behind other writers to the same space; those writers now wait for its upload instead. Details: TC-2954. - A SQL checkpoint save over 8 MiB reads its row's metadata before opening the transaction that uploads it, so that transaction holds no table lock while the payload is sent. A pooled connection closed because it was released inside a transaction is now logged as a warning. Details: TC-2954. +- SQL: a database whose stored checkpoint or WAL delta is larger than 64 MiB is no longer loaded; requests to it answer `507 sql_database_size_limit` and nothing runs (`storage.sql.max_artifact_read_bytes`, env `TINYCLOUD_STORAGE__SQL__MAX_ARTIFACT_READ_BYTES`; `0` off). At most two database artifact transfers over 8 MiB (whole-checkpoint saves and large loads) run at once per node; further ones wait without holding a database connection. The checkpoint back-off is held in the node's memory: a restart clears it. Details: TC-2961. - SQLite stamp replay deadlines now use the same timestamp encoding as cleanup (TC-2920). - PostgreSQL database artifacts use uncompressed external storage for new writes to avoid repeated decompression during chunked reads (TC-2917). Existing large artifacts require an operator re-toast; verify both storage settings after rollout and follow `scripts/tc2917-README.md` if lock contention deferred the migration. - Encryption: v2 column format with key ids, per-purpose keys and recorded format metadata. Optional strict reads are off by default. Details: TC-2516, TC-2517. diff --git a/docs/sql-local-durability.md b/docs/sql-local-durability.md index 595f1f6e..aa43a996 100644 --- a/docs/sql-local-durability.md +++ b/docs/sql-local-durability.md @@ -61,10 +61,13 @@ A kill -9 test cannot inject "after COMMIT, before its fsync completed"; `before | `max_database_bytes` | `0` (no cap) | per-database size cap, enforced by SQLite's `max_page_count` | | `min_free_bytes` | `0` (check off) | disk kept free after all in-flight reservations; `0` disables the gate, the reservation and the free-space checks before COMMIT (a full-disk error still reads free space once, see Errors) | | `max_artifact_write_bytes` | `64 MiB` (on) | the largest SQL or DuckDB artifact payload saved in one statement (TC-2961). A larger whole checkpoint is refused before it is sent, with `507 sql_database_size_limit`, and the local change is discarded. `0` turns it off, for local tests only | +| `max_artifact_read_bytes` | `64 MiB` (on) | the largest stored SQL or DuckDB checkpoint or WAL delta the node loads (TC-2961). A larger database is refused before any of it is read, with `507 sql_database_size_limit`, instead of being loaded whole. `0` turns it off | **The two limits are off by default.** They are Sam's decision (card `sql-storage-report-read`) and are set by config once he answers; an unmeasured cap could refuse real users' databases. The mechanism is fully tested with explicit values. What stays on by default besides `synchronous`: the transaction per write request and the owner-budget limit below. -**The artifact write limit is on by default (TC-2961).** A whole checkpoint is saved as one parameter of one statement, and PostgreSQL needs a multiple of it in memory, so one tenant's large database could take down the shared primary. A database whose checkpoint exceeds 64 MiB cannot persist a write until artifacts are stored in bounded parts (TC-2917). After a whole-checkpoint save fails or is refused, the database's writes are refused without reloading it (`507` after a refusal, `503 unavailable_retryable` after a failure) for 30 s, doubling per further failure up to 15 min; reads are not affected, and a successful checkpoint save ends the back-off. +**The artifact write limit is on by default (TC-2961).** A whole checkpoint is saved as one parameter of one statement, and PostgreSQL needs a multiple of it in memory, so one tenant's large database could take down the shared primary. A database whose checkpoint exceeds 64 MiB cannot persist a write until artifacts are stored in bounded parts (TC-2917). After a whole-checkpoint save fails or is refused, the database's writes are refused without reloading it (`507` after a refusal, `503 unavailable_retryable` after a failure) for 30 s, doubling per further failure up to 15 min; reads are not affected, and a successful checkpoint save ends the back-off. The back-off is held in the node's memory: a restart clears it, and each node instance keeps its own. + +**The artifact read limit is on by default (TC-2961).** A database whose stored checkpoint or WAL delta exceeds 64 MiB is not loaded: every request to it is answered `507 sql_database_size_limit` until artifacts are stored in bounded parts (TC-2917). At most two artifact transfers over 8 MiB (whole-checkpoint saves and loads of a large database) run at once per node; further ones wait, holding no database connection, within their request's bounds. **Owner budget is not switchable and can refuse a write today's code allows.** `max_page_count` is set from the owner's remaining quota (`limit - used`, the same `storage_check` read as before). Today a write that overshoots a non-zero remainder is allowed once and only the next one is refused with 402; now the overshooting write itself is refused with 402 before COMMIT. The comparison is the database's page growth against the remainder, the unit the quota already charges (logical database size); growth that frees pages elsewhere in the file is not counted, so it is never stricter than the charge. diff --git a/tinycloud-core/src/database_artifacts.rs b/tinycloud-core/src/database_artifacts.rs index 8a143563..f81196b6 100644 --- a/tinycloud-core/src/database_artifacts.rs +++ b/tinycloud-core/src/database_artifacts.rs @@ -111,6 +111,10 @@ pub enum DatabaseArtifactError { /// (TC-2961). Refused before any statement; nothing was written. #[error("database artifact payload of {bytes} bytes exceeds the {limit}-byte write limit")] WriteLimitExceeded { bytes: u64, limit: u64 }, + /// A stored part larger than the repository's read limit (TC-2961). + /// Refused before any payload byte was read. + #[error("database artifact {part} exceeds the read limit")] + ReadLimitExceeded { part: &'static str }, #[error("database artifact backend error: {0}")] Backend(String), #[error("database artifact checkpoint is missing")] @@ -678,6 +682,16 @@ pub const ARTIFACT_READ_RETRIES: usize = 2; /// until artifacts are stored in bounded parts (TC-2917). pub const DEFAULT_MAX_ARTIFACT_WRITE_BYTES: u64 = 64 * 1024 * 1024; +/// The default largest stored part (checkpoint or WAL delta) a read loads +/// (TC-2961): a larger database is refused instead of being loaded whole. +pub const DEFAULT_MAX_ARTIFACT_READ_BYTES: u64 = 64 * 1024 * 1024; + +/// How many artifact transfers larger than [`BASE_TIMEOUT_PAYLOAD_BYTES`] +/// (whole-checkpoint saves, loads of a large database) one repository runs +/// at once (TC-2961), so several large databases cannot stack their memory +/// on the shared primary. Further ones wait, holding no database connection. +pub const MAX_CONCURRENT_LARGE_TRANSFERS: usize = 2; + /// The stored metadata a full checkpoint save is checked against. struct CurrentArtifact { revision: i64, @@ -691,6 +705,11 @@ pub struct SeaOrmDatabaseArtifactRepository { sqlite_writer_lock: Option>>, /// The largest payload a save sends; 0 = no limit (local tests only). max_write_bytes: u64, + /// The largest stored part a read loads; 0 = no limit. + max_read_bytes: u64, + /// Permits for transfers over [`BASE_TIMEOUT_PAYLOAD_BYTES`]; shared by + /// every clone of this repository. + large_transfers: std::sync::Arc, /// Test-only rendezvous seam (see [`wait_at_race_barrier`]) that lets two /// writers read the same base revision before either commits, so the /// full-checkpoint CAS conflict path can be exercised deterministically. @@ -704,6 +723,10 @@ impl SeaOrmDatabaseArtifactRepository { conn: conn.into(), sqlite_writer_lock: None, max_write_bytes: DEFAULT_MAX_ARTIFACT_WRITE_BYTES, + max_read_bytes: DEFAULT_MAX_ARTIFACT_READ_BYTES, + large_transfers: std::sync::Arc::new(tokio::sync::Semaphore::new( + MAX_CONCURRENT_LARGE_TRANSFERS, + )), #[cfg(test)] race_barrier: None, } @@ -726,6 +749,26 @@ impl SeaOrmDatabaseArtifactRepository { self } + /// Refuse reads of a database whose stored checkpoint or WAL delta is + /// larger than `max_read_bytes` (TC-2961); 0 turns the limit off. + pub fn with_max_read_bytes(mut self, max_read_bytes: u64) -> Self { + self.max_read_bytes = max_read_bytes; + self + } + + /// A permit for a transfer of `bytes`, when it is a large one. + async fn large_transfer_permit(&self, bytes: u64) -> Option> { + if bytes <= BASE_TIMEOUT_PAYLOAD_BYTES as u64 { + return None; + } + Some( + self.large_transfers + .acquire() + .await + .expect("the large-transfer semaphore is never closed"), + ) + } + fn check_write(&self, bytes: usize) -> Result<(), DatabaseArtifactError> { let bytes = bytes as u64; if self.max_write_bytes != 0 && bytes > self.max_write_bytes { @@ -919,6 +962,25 @@ impl DatabaseArtifactRepository for SeaOrmDatabaseArtifactRepository { meta: &ArtifactMeta, sink: &mut dyn ArtifactSink, ) -> Result<(), DatabaseArtifactError> { + // TC-2961: refuse an oversized database before any slice is read. + if self.max_read_bytes != 0 { + for (part, bytes) in [ + (ArtifactPart::Checkpoint, meta.checkpoint_size_bytes), + (ArtifactPart::Wal, meta.delta_size_bytes), + ] { + if u64::try_from(bytes).unwrap_or(0) > self.max_read_bytes { + return Err(DatabaseArtifactError::ReadLimitExceeded { + part: part.as_str(), + }); + } + } + } + let total = u64::try_from( + meta.checkpoint_size_bytes + .saturating_add(meta.delta_size_bytes), + ) + .unwrap_or(0); + let _large = self.large_transfer_permit(total).await; let key = ArtifactKey { service, space, @@ -967,6 +1029,8 @@ impl DatabaseArtifactRepository for SeaOrmDatabaseArtifactRepository { ) -> Result { // TC-2961: before the writer lock, the hash and any statement. self.check_write(payload.len())?; + // Before the writer lock: a waiting save holds nothing else. + let _large = self.large_transfer_permit(payload.len() as u64).await; let _writer = match &self.sqlite_writer_lock { Some(lock) => Some(lock.lock().await), None => None, @@ -1577,6 +1641,188 @@ mod tests { assert_eq!((meta.revision, meta.delta_size_bytes), (1, 0)); } + /// TC-2961: a stored checkpoint or WAL delta over the read limit is + /// refused before the sink receives anything; one at the limit is read. + #[tokio::test] + async fn a_part_over_the_read_limit_is_refused_before_anything_is_read() { + let conn = Database::connect(ConnectOptions::new("sqlite::memory:".to_string())) + .await + .unwrap(); + Migrator::up(&conn, None).await.unwrap(); + assert_eq!( + SeaOrmDatabaseArtifactRepository::new(conn.clone()).max_read_bytes, + DEFAULT_MAX_ARTIFACT_READ_BYTES + ); + let repo = SeaOrmDatabaseArtifactRepository::new(conn).with_max_read_bytes(4096); + let read = |name: &'static str| { + let repo = repo.clone(); + async move { + let meta = repo.load_meta("sql", "space", name).await.unwrap().unwrap(); + let mut sink = MemorySink::default(); + let result = repo + .read_chunks("sql", "space", name, &meta, &mut sink) + .await; + (result, sink) + } + }; + + repo.save( + "sql", + "space", + "big", + vec![1; 4097], + ArtifactExpectation::Absent, + ) + .await + .unwrap(); + let (result, sink) = read("big").await; + assert!( + matches!( + result, + Err(DatabaseArtifactError::ReadLimitExceeded { part: "checkpoint" }) + ), + "{result:?}" + ); + assert!(sink.checkpoint.is_empty() && sink.wal.is_none()); + + let saved = repo + .save( + "sql", + "space", + "wal", + vec![1; 4096], + ArtifactExpectation::Absent, + ) + .await + .unwrap(); + let (result, sink) = read("wal").await; + result.unwrap(); + assert_eq!(sink.checkpoint.len(), 4096, "at the limit is read"); + repo.save_delta( + "sql", + "space", + "wal", + vec![2; 4097], + ArtifactExpectation::Derived { + revision: saved.revision, + checkpoint_content_hash: saved.checkpoint_content_hash, + }, + ) + .await + .unwrap(); + let (result, sink) = read("wal").await; + assert!( + matches!( + result, + Err(DatabaseArtifactError::ReadLimitExceeded { part: "wal" }) + ), + "{result:?}" + ); + assert!(sink.checkpoint.is_empty() && sink.wal.is_none()); + } + + /// Holds every transfer at its first part until the gate opens. + struct GatedSink { + entered: std::sync::Arc, + gate: std::sync::Arc, + } + + #[async_trait] + impl ArtifactSink for GatedSink { + async fn reset(&mut self) -> Result<(), DatabaseArtifactError> { + Ok(()) + } + async fn begin_part(&mut self, part: ArtifactPart) -> Result<(), DatabaseArtifactError> { + if part == ArtifactPart::Checkpoint { + self.entered + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + drop(self.gate.acquire().await.unwrap()); + } + Ok(()) + } + async fn write(&mut self, _bytes: &[u8]) -> Result<(), DatabaseArtifactError> { + Ok(()) + } + async fn finish_part(&mut self) -> Result<(), DatabaseArtifactError> { + Ok(()) + } + } + + /// TC-2961: at most [`MAX_CONCURRENT_LARGE_TRANSFERS`] transfers over + /// 8 MiB run at once; the next one starts when one of them ends, and a + /// small one is never held back by them. + #[tokio::test] + async fn large_transfers_run_at_most_two_at_a_time() { + use std::sync::{atomic::Ordering, Arc}; + + let conn = Database::connect(ConnectOptions::new("sqlite::memory:".to_string())) + .await + .unwrap(); + Migrator::up(&conn, None).await.unwrap(); + let repo = SeaOrmDatabaseArtifactRepository::new(conn); + let large = BASE_TIMEOUT_PAYLOAD_BYTES + 1; + repo.save( + "sql", + "space", + "large", + vec![1; large], + ArtifactExpectation::Absent, + ) + .await + .unwrap(); + repo.save( + "sql", + "space", + "small", + vec![1; 16], + ArtifactExpectation::Absent, + ) + .await + .unwrap(); + let entered = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let read = |name: &'static str| { + let repo = repo.clone(); + let mut sink = GatedSink { + entered: entered.clone(), + gate: gate.clone(), + }; + tokio::spawn(async move { + let meta = repo.load_meta("sql", "space", name).await.unwrap().unwrap(); + repo.read_chunks("sql", "space", name, &meta, &mut sink) + .await + }) + }; + let wait_for = |count: usize| { + let entered = entered.clone(); + async move { + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while entered.load(Ordering::SeqCst) < count { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("transfers did not start"); + } + }; + + let readers: Vec<_> = (0..3).map(|_| read("large")).collect(); + wait_for(2).await; + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert_eq!(entered.load(Ordering::SeqCst), 2, "the third waits"); + let small = read("small"); + wait_for(3).await; + + // Each sink hands its gate permit back, so one lets every transfer + // through in turn; the third large one enters when a slot frees. + gate.add_permits(1); + small.await.unwrap().unwrap(); + for reader in readers { + reader.await.unwrap().unwrap(); + } + assert_eq!(entered.load(Ordering::SeqCst), 4); + } + #[tokio::test] async fn postgres_absent_create_refuses_an_existing_or_racing_row() { use sea_orm::{ConnectionTrait, DbBackend, Statement, TransactionTrait}; diff --git a/tinycloud-core/src/sql/service.rs b/tinycloud-core/src/sql/service.rs index be61e3a6..96e17ef0 100644 --- a/tinycloud-core/src/sql/service.rs +++ b/tinycloud-core/src/sql/service.rs @@ -52,6 +52,8 @@ enum HydrationOutcome { Unavailable(&'static str), Failed(String), TimedOut, + /// The stored database exceeds the artifact read limit (TC-2961). + TooLarge, } const PHASE_LOADING: u8 = 0; @@ -729,6 +731,7 @@ impl SqlService { return Err(SqlError::Unavailable); } HydrationOutcome::Failed(message) => return Err(SqlError::Internal(message)), + HydrationOutcome::TooLarge => return Err(SqlError::DatabaseTooLargeToLoad), HydrationOutcome::TimedOut => { log_busy( SqlBusyStage::ArtifactTransfer, @@ -1225,6 +1228,14 @@ fn artifact_error_to_sql(err: DatabaseArtifactError) -> SqlError { ); SqlError::DatabaseSizeLimit } + DatabaseArtifactError::ReadLimitExceeded { .. } => { + tracing::warn!( + service = "sql", + error_class = "artifact_read_limit", + "Database exceeds the artifact read limit; not loaded" + ); + SqlError::DatabaseTooLargeToLoad + } err => SqlError::Internal(err.to_string()), } } @@ -1254,6 +1265,7 @@ fn hydration_failure(error: SqlError, kind: &'static str) -> HydrationOutcome { match error { SqlError::Unavailable => HydrationOutcome::Unavailable(kind), SqlError::Internal(message) => HydrationOutcome::Failed(message), + SqlError::DatabaseTooLargeToLoad => HydrationOutcome::TooLarge, error => HydrationOutcome::Failed(error.to_string()), } } @@ -3102,6 +3114,66 @@ mod tests { assert_eq!(loads.load(Ordering::Relaxed), 3); } + /// TC-2961: a database stored over the artifact read limit is not + /// loaded; a read and a write to it both answer the 507 size limit, not + /// an internal error. + #[tokio::test] + async fn a_database_over_the_read_limit_answers_the_size_limit() { + let db = Database::connect(ConnectOptions::new("sqlite::memory:".to_string())) + .await + .unwrap(); + Migrator::up(&db, None).await.unwrap(); + let space = test_space_id("sql-read-limit"); + let insert = || SqlRequest::Execute { + schema: Some(vec!["CREATE TABLE IF NOT EXISTS items (v)".to_string()]), + sql: "INSERT INTO items VALUES (1)".to_string(), + params: Vec::new(), + }; + let admin = || "tinycloud.sql/admin".to_string(); + let writer_cache = TempDir::new().unwrap(); + let writer = SqlService::new( + writer_cache.path().to_string_lossy().to_string(), + u64::MAX, + Arc::new(SeaOrmDatabaseArtifactRepository::new(db.clone())), + ); + writer + .execute(&space, "main", insert(), None, admin()) + .await + .unwrap(); + + let reader_cache = TempDir::new().unwrap(); + let reader = SqlService::new( + reader_cache.path().to_string_lossy().to_string(), + u64::MAX, + Arc::new(SeaOrmDatabaseArtifactRepository::new(db).with_max_read_bytes(1)), + ); + let read = reader + .execute( + &space, + "main", + SqlRequest::Query { + sql: "SELECT count(*) FROM items".into(), + params: vec![], + max_rows: None, + max_bytes: None, + }, + None, + "tinycloud.sql/read".to_string(), + ) + .await; + assert!( + matches!(read, Err(SqlError::DatabaseTooLargeToLoad)), + "{read:?}" + ); + let written = reader + .execute(&space, "main", insert(), None, admin()) + .await; + assert!( + matches!(written, Err(SqlError::DatabaseTooLargeToLoad)), + "{written:?}" + ); + } + /// Fails the next `failures` repository calls with `error`, then /// delegates. struct FlakyArtifactRepository { diff --git a/tinycloud-core/src/sql/types.rs b/tinycloud-core/src/sql/types.rs index 85ebf299..55fd0a70 100644 --- a/tinycloud-core/src/sql/types.rs +++ b/tinycloud-core/src/sql/types.rs @@ -346,6 +346,11 @@ pub enum SqlError { /// (TC-2961). #[error("sql_database_size_limit: the write would exceed the database size limit; no change was applied")] DatabaseSizeLimit, + /// The database is larger than the node loads (TC-2961): its stored + /// checkpoint or WAL delta exceeds the artifact read limit, so it was + /// not loaded and nothing ran. + #[error("sql_database_size_limit: the database is larger than this node loads; no change was applied")] + DatabaseTooLargeToLoad, /// The node's disk cannot hold the write's worst-case growth next to the /// writes already in flight. Rolled back before COMMIT; nothing was /// applied, and a later retry may succeed (TC-1491). diff --git a/tinycloud-node-server/src/config.rs b/tinycloud-node-server/src/config.rs index f7fc8bd4..6e55df7a 100644 --- a/tinycloud-node-server/src/config.rs +++ b/tinycloud-node-server/src/config.rs @@ -1661,6 +1661,12 @@ pub struct SqlStorageConfig { /// off, for local tests only. #[serde(default = "default_sql_max_artifact_write_bytes")] pub max_artifact_write_bytes: ByteUnit, + /// The largest stored SQL or DuckDB checkpoint or WAL delta the node + /// loads (TC-2961). A larger database is refused before any of it is + /// read (`507 sql_database_size_limit`) instead of being loaded whole. + /// On by default, 64 MiB; 0 turns it off. + #[serde(default = "default_sql_max_artifact_read_bytes")] + pub max_artifact_read_bytes: ByteUnit, } fn default_sql_max_database_bytes() -> ByteUnit { @@ -1675,6 +1681,10 @@ fn default_sql_max_artifact_write_bytes() -> ByteUnit { ByteUnit::Byte(tinycloud_core::database_artifacts::DEFAULT_MAX_ARTIFACT_WRITE_BYTES) } +fn default_sql_max_artifact_read_bytes() -> ByteUnit { + ByteUnit::Byte(tinycloud_core::database_artifacts::DEFAULT_MAX_ARTIFACT_READ_BYTES) +} + fn default_sql_memory_threshold() -> ByteUnit { ByteUnit::Mebibyte(10) } @@ -1730,6 +1740,7 @@ impl Default for SqlStorageConfig { min_free_bytes: default_sql_min_free_bytes(), ddl_requires_schema: DdlPolicy::default(), max_artifact_write_bytes: default_sql_max_artifact_write_bytes(), + max_artifact_read_bytes: default_sql_max_artifact_read_bytes(), } } } diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 3b43df0a..5d11c67a 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -971,13 +971,21 @@ pub async fn app_with_control( let database_artifact_repository: Arc = Arc::new(SizeTrackingArtifactRepository::new( Arc::new( - node_artifact_repository(&tinycloud).with_max_write_bytes( - tinycloud_config - .storage - .sql - .max_artifact_write_bytes - .as_u64(), - ), + node_artifact_repository(&tinycloud) + .with_max_write_bytes( + tinycloud_config + .storage + .sql + .max_artifact_write_bytes + .as_u64(), + ) + .with_max_read_bytes( + tinycloud_config + .storage + .sql + .max_artifact_read_bytes + .as_u64(), + ), ), sql_sizes.clone(), )); diff --git a/tinycloud-node-server/src/routes/mod.rs b/tinycloud-node-server/src/routes/mod.rs index f103817c..e478c38d 100644 --- a/tinycloud-node-server/src/routes/mod.rs +++ b/tinycloud-node-server/src/routes/mod.rs @@ -3661,7 +3661,9 @@ pub(crate) fn sql_error_to_status(err: &SqlError) -> Status { // 507, not 413: the SQL client reads 413 as `SQL_RESPONSE_TOO_LARGE` // and the KV and quota clients as an exhausted storage plan, and // TC-1012's 413 is the request body cap. Neither is true here. - SqlError::DatabaseSizeLimit | SqlError::DiskHeadroom => Status::new(507), + SqlError::DatabaseSizeLimit | SqlError::DatabaseTooLargeToLoad | SqlError::DiskHeadroom => { + Status::new(507) + } SqlError::InvalidStatement(_) => Status::BadRequest, SqlError::SchemaError(_) => Status::BadRequest, SqlError::ReadOnlyViolation | SqlError::CaveatViolation(_) => Status::Forbidden, From 638d3a3195eae676434faa03cd8ed05757352a41 Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:57:13 +0000 Subject: [PATCH 3/5] docs(TC-2954): name what waits behind a protected legacy upload on PostgreSQL --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3170614f..a43a8eca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - A registered Policy/v3 root never authorizes a request by itself, whether the invocation cites it alone or beside other proofs, and whichever codec or multibase encoding of its CID it uses: policy roots are matched by their canonical identity. `/invoke` and the routes that share its policy check, the encryption network and key routes, and `POST /policy/v3/deliveries/authorize` answer 403. Access under a policy needs a session minted under the policy's conditions. Details: TC-2958. - `POST /policy/v3/deliveries/authorize` runs the same policy check as `/invoke`: an invocation under a policy session must meet the session's conditions (a single proof, an invocation lifetime of at most 60 seconds, its scope and liveness), and a session authorizes delivery only of a share under its own policy; otherwise the answer is 403 `delivery-authorization-invalid`. Details: TC-2958. - A protected legacy meeting write now waits for its space's write order before opening its transaction on PostgreSQL too (as on SQLite), so it no longer holds a pooled connection while queueing behind other writers to the same space; those writers now wait for its upload instead. Details: TC-2954. +- A protected legacy meeting write now waits for its space's write order before opening its transaction on PostgreSQL too (as on SQLite), so it no longer holds a pooled connection while queueing behind other writers to the same space. In exchange, on PostgreSQL it now holds that space's write order across its upload, up to 240 s: every other write to the same space waits for it meanwhile, including ordinary KV writes, delegations and revocations, epoch-head repair, Policy/v3 registration and session admission. Other spaces are not affected, and a client cannot slow the upload (the body is staged first). An ordinary KV write that waits long enough may be refused as `KvCandidateUnavailable` (it fails closed). Details: TC-2954. - A SQL checkpoint save over 8 MiB reads its row's metadata before opening the transaction that uploads it, so that transaction holds no table lock while the payload is sent. A pooled connection closed because it was released inside a transaction is now logged as a warning. Details: TC-2954. - SQL: a database whose stored checkpoint or WAL delta is larger than 64 MiB is no longer loaded; requests to it answer `507 sql_database_size_limit` and nothing runs (`storage.sql.max_artifact_read_bytes`, env `TINYCLOUD_STORAGE__SQL__MAX_ARTIFACT_READ_BYTES`; `0` off). At most two database artifact transfers over 8 MiB (whole-checkpoint saves and large loads) run at once per node; further ones wait without holding a database connection. The checkpoint back-off is held in the node's memory: a restart clears it. Details: TC-2961. - SQLite stamp replay deadlines now use the same timestamp encoding as cleanup (TC-2920). From 3362fe54292145baaf6449a39fb5ca4369e9d3af Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:07:24 +0000 Subject: [PATCH 4/5] docs(TC-2961): with_checkpoint_backoff is for tests; wiring it is a behaviour change --- CHANGELOG.md | 3 --- tinycloud-core/src/sql/service.rs | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a43a8eca..e3bd061d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,6 @@ ## [Unreleased] ### Changes - PostgreSQL: every pooled connection sets `idle_in_transaction_session_timeout` (`database.idle_in_transaction_timeout_ms`, env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`, default 300000 = 5 min, `0` off), so the server ends a transaction left idle that long and the pool replaces the session. A non-zero value of 240 s or less is refused at startup: a protected legacy meeting write legitimately holds its transaction across an upload of up to 240 s. It is not set in `statement_timeout_mode = "local"` (transaction-pooling proxies). `database.idle_in_transaction_timeout_ms`, `statement_timeout_ms` and `bulk_statement_timeout_ms` above 2147483647 (PostgreSQL's largest) are now refused at startup instead of failing every connection. Details: TC-2954. -- A registered Policy/v3 root never authorizes a request by itself, whether the invocation cites it alone or beside other proofs, and whichever codec or multibase encoding of its CID it uses: policy roots are matched by their canonical identity. `/invoke` and the routes that share its policy check, the encryption network and key routes, and `POST /policy/v3/deliveries/authorize` answer 403. Access under a policy needs a session minted under the policy's conditions. Details: TC-2958. -- `POST /policy/v3/deliveries/authorize` runs the same policy check as `/invoke`: an invocation under a policy session must meet the session's conditions (a single proof, an invocation lifetime of at most 60 seconds, its scope and liveness), and a session authorizes delivery only of a share under its own policy; otherwise the answer is 403 `delivery-authorization-invalid`. Details: TC-2958. -- A protected legacy meeting write now waits for its space's write order before opening its transaction on PostgreSQL too (as on SQLite), so it no longer holds a pooled connection while queueing behind other writers to the same space; those writers now wait for its upload instead. Details: TC-2954. - A protected legacy meeting write now waits for its space's write order before opening its transaction on PostgreSQL too (as on SQLite), so it no longer holds a pooled connection while queueing behind other writers to the same space. In exchange, on PostgreSQL it now holds that space's write order across its upload, up to 240 s: every other write to the same space waits for it meanwhile, including ordinary KV writes, delegations and revocations, epoch-head repair, Policy/v3 registration and session admission. Other spaces are not affected, and a client cannot slow the upload (the body is staged first). An ordinary KV write that waits long enough may be refused as `KvCandidateUnavailable` (it fails closed). Details: TC-2954. - A SQL checkpoint save over 8 MiB reads its row's metadata before opening the transaction that uploads it, so that transaction holds no table lock while the payload is sent. A pooled connection closed because it was released inside a transaction is now logged as a warning. Details: TC-2954. - SQL: a database whose stored checkpoint or WAL delta is larger than 64 MiB is no longer loaded; requests to it answer `507 sql_database_size_limit` and nothing runs (`storage.sql.max_artifact_read_bytes`, env `TINYCLOUD_STORAGE__SQL__MAX_ARTIFACT_READ_BYTES`; `0` off). At most two database artifact transfers over 8 MiB (whole-checkpoint saves and large loads) run at once per node; further ones wait without holding a database connection. The checkpoint back-off is held in the node's memory: a restart clears it. Details: TC-2961. diff --git a/tinycloud-core/src/sql/service.rs b/tinycloud-core/src/sql/service.rs index 96e17ef0..12c7bec2 100644 --- a/tinycloud-core/src/sql/service.rs +++ b/tinycloud-core/src/sql/service.rs @@ -232,7 +232,10 @@ impl SqlService { /// The checkpoint back-off (TC-2961): `first` after a failed or refused /// whole-checkpoint save, doubling per further failure up to `max`. The - /// node keeps the defaults (30 s, 15 min); only tests lower them. + /// node keeps the defaults (30 s, 15 min); only tests lower them. Public + /// only for the node-server tests: wiring it to node config is a + /// behaviour change that needs review, since a zero back-off brings back + /// the reload-and-retry loop TC-2961 removed. pub fn with_checkpoint_backoff(mut self, first: Duration, max: Duration) -> Self { self.checkpoint_backoff_bounds = (first, max); self From 2eab157d84bca581a662cf8b86aff0588d998eae Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:21:11 +0000 Subject: [PATCH 5/5] fix(TC-2961): no back-off for a save still queued for its transfer slot; DuckDB 507; cap cross-check; review tests --- CHANGELOG.md | 2 +- docs/sql-local-durability.md | 14 +- tinycloud-core/src/database_artifacts.rs | 277 ++++++++++++++++++ tinycloud-core/src/duckdb/service.rs | 7 +- tinycloud-core/src/duckdb/types.rs | 8 + tinycloud-core/src/sql/service.rs | 139 ++++++++- tinycloud-core/tests/artifact_bounded_read.rs | 3 +- tinycloud-node-server/src/config.rs | 41 ++- tinycloud-node-server/src/lib.rs | 5 + tinycloud-node-server/src/routes/mod.rs | 21 ++ 10 files changed, 503 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3bd061d..ba3c8be6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - PostgreSQL: every pooled connection sets `idle_in_transaction_session_timeout` (`database.idle_in_transaction_timeout_ms`, env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`, default 300000 = 5 min, `0` off), so the server ends a transaction left idle that long and the pool replaces the session. A non-zero value of 240 s or less is refused at startup: a protected legacy meeting write legitimately holds its transaction across an upload of up to 240 s. It is not set in `statement_timeout_mode = "local"` (transaction-pooling proxies). `database.idle_in_transaction_timeout_ms`, `statement_timeout_ms` and `bulk_statement_timeout_ms` above 2147483647 (PostgreSQL's largest) are now refused at startup instead of failing every connection. Details: TC-2954. - A protected legacy meeting write now waits for its space's write order before opening its transaction on PostgreSQL too (as on SQLite), so it no longer holds a pooled connection while queueing behind other writers to the same space. In exchange, on PostgreSQL it now holds that space's write order across its upload, up to 240 s: every other write to the same space waits for it meanwhile, including ordinary KV writes, delegations and revocations, epoch-head repair, Policy/v3 registration and session admission. Other spaces are not affected, and a client cannot slow the upload (the body is staged first). An ordinary KV write that waits long enough may be refused as `KvCandidateUnavailable` (it fails closed). Details: TC-2954. - A SQL checkpoint save over 8 MiB reads its row's metadata before opening the transaction that uploads it, so that transaction holds no table lock while the payload is sent. A pooled connection closed because it was released inside a transaction is now logged as a warning. Details: TC-2954. -- SQL: a database whose stored checkpoint or WAL delta is larger than 64 MiB is no longer loaded; requests to it answer `507 sql_database_size_limit` and nothing runs (`storage.sql.max_artifact_read_bytes`, env `TINYCLOUD_STORAGE__SQL__MAX_ARTIFACT_READ_BYTES`; `0` off). At most two database artifact transfers over 8 MiB (whole-checkpoint saves and large loads) run at once per node; further ones wait without holding a database connection. The checkpoint back-off is held in the node's memory: a restart clears it. Details: TC-2961. +- SQL and DuckDB: a database whose stored checkpoint or WAL delta is larger than 64 MiB is no longer loaded; requests to it answer `507 sql_database_size_limit` and nothing runs (`storage.sql.max_artifact_read_bytes`, env `TINYCLOUD_STORAGE__SQL__MAX_ARTIFACT_READ_BYTES`; `0` off; the node refuses to start with it below the write limit). A database already stored over the limit becomes unreadable, not only unwritable. DuckDB now answers both artifact limits `507` instead of `500`. At most two database artifact transfers over 8 MiB (whole-checkpoint saves and large loads) run at once per node, for all tenants; further ones wait without holding a database connection, and a SQL save cut while still waiting starts no write back-off. The checkpoint back-off is held in the node's memory: a restart clears it. Details: TC-2961. - SQLite stamp replay deadlines now use the same timestamp encoding as cleanup (TC-2920). - PostgreSQL database artifacts use uncompressed external storage for new writes to avoid repeated decompression during chunked reads (TC-2917). Existing large artifacts require an operator re-toast; verify both storage settings after rollout and follow `scripts/tc2917-README.md` if lock contention deferred the migration. - Encryption: v2 column format with key ids, per-purpose keys and recorded format metadata. Optional strict reads are off by default. Details: TC-2516, TC-2517. diff --git a/docs/sql-local-durability.md b/docs/sql-local-durability.md index aa43a996..e750705b 100644 --- a/docs/sql-local-durability.md +++ b/docs/sql-local-durability.md @@ -61,13 +61,19 @@ A kill -9 test cannot inject "after COMMIT, before its fsync completed"; `before | `max_database_bytes` | `0` (no cap) | per-database size cap, enforced by SQLite's `max_page_count` | | `min_free_bytes` | `0` (check off) | disk kept free after all in-flight reservations; `0` disables the gate, the reservation and the free-space checks before COMMIT (a full-disk error still reads free space once, see Errors) | | `max_artifact_write_bytes` | `64 MiB` (on) | the largest SQL or DuckDB artifact payload saved in one statement (TC-2961). A larger whole checkpoint is refused before it is sent, with `507 sql_database_size_limit`, and the local change is discarded. `0` turns it off, for local tests only | -| `max_artifact_read_bytes` | `64 MiB` (on) | the largest stored SQL or DuckDB checkpoint or WAL delta the node loads (TC-2961). A larger database is refused before any of it is read, with `507 sql_database_size_limit`, instead of being loaded whole. `0` turns it off | +| `max_artifact_read_bytes` | `64 MiB` (on) | the largest stored SQL or DuckDB checkpoint or WAL delta the node loads (TC-2961). A larger database is refused before any of it is read, with `507 sql_database_size_limit`, instead of being loaded whole. `0` turns it off; otherwise the node refuses to start with it below `max_artifact_write_bytes` (or with that off), since databases between the two would be saved and then unloadable | -**The two limits are off by default.** They are Sam's decision (card `sql-storage-report-read`) and are set by config once he answers; an unmeasured cap could refuse real users' databases. The mechanism is fully tested with explicit values. What stays on by default besides `synchronous`: the transaction per write request and the owner-budget limit below. +**`max_database_bytes` and `min_free_bytes` are off by default.** They are Sam's decision (card `sql-storage-report-read`) and are set by config once he answers; an unmeasured cap could refuse real users' databases. The mechanism is fully tested with explicit values. What stays on by default besides `synchronous`: the transaction per write request, the owner-budget limit and the two artifact limits below. -**The artifact write limit is on by default (TC-2961).** A whole checkpoint is saved as one parameter of one statement, and PostgreSQL needs a multiple of it in memory, so one tenant's large database could take down the shared primary. A database whose checkpoint exceeds 64 MiB cannot persist a write until artifacts are stored in bounded parts (TC-2917). After a whole-checkpoint save fails or is refused, the database's writes are refused without reloading it (`507` after a refusal, `503 unavailable_retryable` after a failure) for 30 s, doubling per further failure up to 15 min; reads are not affected, and a successful checkpoint save ends the back-off. The back-off is held in the node's memory: a restart clears it, and each node instance keeps its own. +**The artifact write limit is on by default (TC-2961).** A whole checkpoint is saved as one parameter of one statement, and PostgreSQL needs a multiple of it in memory, so one tenant's large database could take down the shared primary. A database whose checkpoint exceeds 64 MiB cannot persist a write until artifacts are stored in bounded parts (TC-2917). After a whole-checkpoint save fails or is refused, the database's writes are refused without reloading it (`507` after a refusal, `503 unavailable_retryable` after a failure) for 30 s, doubling per further failure up to 15 min; reads are not affected, and a successful checkpoint save ends the back-off. A save cut while it still waits for its large-transfer slot (below) sent nothing and starts no back-off. The back-off is held in the node's memory: a restart clears it, and each node instance keeps its own. -**The artifact read limit is on by default (TC-2961).** A database whose stored checkpoint or WAL delta exceeds 64 MiB is not loaded: every request to it is answered `507 sql_database_size_limit` until artifacts are stored in bounded parts (TC-2917). At most two artifact transfers over 8 MiB (whole-checkpoint saves and loads of a large database) run at once per node; further ones wait, holding no database connection, within their request's bounds. +**The artifact read limit is on by default (TC-2961).** A database whose stored checkpoint or WAL delta exceeds 64 MiB is not loaded: every request to it is answered `507 sql_database_size_limit` until artifacts are stored in bounded parts (TC-2917). A database already stored over the limit becomes unreadable, not only unwritable; count such rows before deploying the limit or lowering it. + +**At most two artifact transfers over 8 MiB run at once per node (TC-2961)**, shared by every tenant and by SQL and DuckDB, first come first served, with no per-tenant share. Further ones wait holding no database connection. How long one can wait and run depends on its path: +- a SQL checkpoint save: wait and upload together within `artifact_transfer_timeout_ms` (45 s); a save cut while still waiting answers `503` and starts no back-off; +- a SQL load: wait and download together within `hydration_max_ms` (600 s), in a task that outlives its callers, who answer `503` after `hydration_wait_timeout_ms` (30 s); +- DuckDB: bounded only by the `/invoke` request deadline where it applies. +Transfers of 8 MiB or less are not counted. **Owner budget is not switchable and can refuse a write today's code allows.** `max_page_count` is set from the owner's remaining quota (`limit - used`, the same `storage_check` read as before). Today a write that overshoots a non-zero remainder is allowed once and only the next one is refused with 402; now the overshooting write itself is refused with 402 before COMMIT. The comparison is the database's page growth against the remainder, the unit the quota already charges (logical database size); growth that frees pages elsewhere in the file is not counted, so it is never stricter than the charge. diff --git a/tinycloud-core/src/database_artifacts.rs b/tinycloud-core/src/database_artifacts.rs index f81196b6..74e91325 100644 --- a/tinycloud-core/src/database_artifacts.rs +++ b/tinycloud-core/src/database_artifacts.rs @@ -1,3 +1,5 @@ +use std::future::Future; + use async_trait::async_trait; use sea_orm::{ sea_query::{Expr, Func}, @@ -692,6 +694,21 @@ pub const DEFAULT_MAX_ARTIFACT_READ_BYTES: u64 = 64 * 1024 * 1024; /// on the shared primary. Further ones wait, holding no database connection. pub const MAX_CONCURRENT_LARGE_TRANSFERS: usize = 2; +tokio::task_local! { + /// Set once a save has its large-transfer slot (or needs none); see + /// [`track_transfer_start`]. + static TRANSFER_STARTED: std::sync::Arc; +} + +/// Run `transfer` and report whether a save inside it got past its wait for +/// a large-transfer slot (TC-2961). A save cut while still queued for a slot +/// sent nothing: it is busy, not a failed checkpoint. +pub async fn track_transfer_start(transfer: F) -> (F::Output, bool) { + let started = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let output = TRANSFER_STARTED.scope(started.clone(), transfer).await; + (output, started.load(std::sync::atomic::Ordering::Acquire)) +} + /// The stored metadata a full checkpoint save is checked against. struct CurrentArtifact { revision: i64, @@ -1031,6 +1048,9 @@ impl DatabaseArtifactRepository for SeaOrmDatabaseArtifactRepository { self.check_write(payload.len())?; // Before the writer lock: a waiting save holds nothing else. let _large = self.large_transfer_permit(payload.len() as u64).await; + let _ = TRANSFER_STARTED.try_with(|started| { + started.store(true, std::sync::atomic::Ordering::Release); + }); let _writer = match &self.sqlite_writer_lock { Some(lock) => Some(lock.lock().await), None => None, @@ -1823,6 +1843,263 @@ mod tests { assert_eq!(entered.load(Ordering::SeqCst), 4); } + /// The node's statement timeouts: a large save opens a real bulk + /// transaction on PostgreSQL (TC-1054). + fn node_statement_timeouts() -> crate::db_pool::StatementTimeouts { + crate::db_pool::StatementTimeouts { + mode: crate::db_pool::StatementTimeoutMode::Local, + base: std::time::Duration::from_secs(30), + bulk: std::time::Duration::from_secs(120), + } + } + + /// TC-2961: two large saves stalled mid-upload (their CAS UPDATE blocked + /// on a row lock another session holds) keep both permits until they + /// finish, so a third large load waits; both permits come back after. + #[tokio::test] + async fn postgres_large_saves_hold_their_permits_through_the_upload() { + use sea_orm::{QueryFilter, QuerySelect, TransactionTrait}; + use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, + }; + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_large_saves_hold_their_permits_through_the_upload", + ) + .await + else { + return; + }; + let conn = crate::db_pool::RecoveringConnection::new(fixture.connect(4).await) + .with_statement_timeouts(node_statement_timeouts()); + let repo = SeaOrmDatabaseArtifactRepository::new(conn); + let large = BASE_TIMEOUT_PAYLOAD_BYTES + 1; + let mut bases = Vec::new(); + for name in ["s1", "s2", "r"] { + let saved = repo + .save( + "sql", + "t", + name, + vec![1; large], + ArtifactExpectation::Absent, + ) + .await + .unwrap(); + bases.push((name, saved.revision, saved.checkpoint_content_hash)); + } + let competitor = fixture.connect(1).await; + let tx = competitor.begin().await.unwrap(); + database_artifact::Entity::find() + .filter(database_artifact::Column::Name.is_in(["s1", "s2"])) + .select_only() + .column(database_artifact::Column::Name) + .lock_exclusive() + .into_tuple::() + .all(&tx) + .await + .unwrap(); + let savers: Vec<_> = bases[..2] + .iter() + .cloned() + .map(|(name, revision, checkpoint_content_hash)| { + let repo = repo.clone(); + tokio::spawn(async move { + repo.save( + "sql", + "t", + name, + vec![2; large], + ArtifactExpectation::Derived { + revision, + checkpoint_content_hash, + }, + ) + .await + }) + }) + .collect(); + tokio::time::sleep(Duration::from_millis(1500)).await; + assert!( + savers.iter().all(|saver| !saver.is_finished()), + "both saves are stalled in their UPDATE" + ); + assert_eq!( + repo.large_transfers.available_permits(), + 0, + "stalled uploads still hold their permits" + ); + let entered = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let reader = { + let repo = repo.clone(); + let mut sink = GatedSink { + entered: entered.clone(), + gate: Arc::new(tokio::sync::Semaphore::new(1)), + }; + tokio::spawn(async move { + let meta = repo.load_meta("sql", "t", "r").await.unwrap().unwrap(); + repo.read_chunks("sql", "t", "r", &meta, &mut sink).await + }) + }; + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!( + entered.load(Ordering::SeqCst), + 0, + "a third large load waits" + ); + tx.rollback().await.unwrap(); + for saver in savers { + tokio::time::timeout(Duration::from_secs(20), saver) + .await + .unwrap() + .unwrap() + .unwrap(); + } + tokio::time::timeout(Duration::from_secs(20), reader) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(repo.large_transfers.available_permits(), 2); + } + + /// TC-2961: with both permits held by stalled large loads, a large save + /// waits, and while it waits a small save on the same ONE-connection + /// pool still runs: the waiter pins no pooled connection and no bulk + /// transaction. + #[tokio::test] + async fn postgres_a_large_save_waiting_for_a_permit_holds_no_connection() { + use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, + }; + let Some(fixture) = crate::test_support::PostgresTestSchema::new( + "postgres_a_large_save_waiting_for_a_permit_holds_no_connection", + ) + .await + else { + return; + }; + let conn = crate::db_pool::RecoveringConnection::new(fixture.connect(1).await) + .with_statement_timeouts(node_statement_timeouts()); + let repo = SeaOrmDatabaseArtifactRepository::new(conn); + let large = BASE_TIMEOUT_PAYLOAD_BYTES + 1; + for name in ["h1", "h2"] { + repo.save( + "sql", + "other", + name, + vec![1; large], + ArtifactExpectation::Absent, + ) + .await + .unwrap(); + } + let entered = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let holders: Vec<_> = ["h1", "h2"] + .into_iter() + .map(|name| { + let repo = repo.clone(); + let mut sink = GatedSink { + entered: entered.clone(), + gate: gate.clone(), + }; + tokio::spawn(async move { + let meta = repo.load_meta("sql", "other", name).await.unwrap().unwrap(); + repo.read_chunks("sql", "other", name, &meta, &mut sink) + .await + }) + }) + .collect(); + tokio::time::timeout(Duration::from_secs(10), async { + while entered.load(Ordering::SeqCst) < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the holders start"); + assert_eq!(repo.large_transfers.available_permits(), 0); + + let saver = { + let repo = repo.clone(); + tokio::spawn(async move { + repo.save( + "sql", + "victim", + "big", + vec![2; large], + ArtifactExpectation::Absent, + ) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(1500)).await; + assert!(!saver.is_finished(), "the third large transfer waits"); + tokio::time::timeout( + Duration::from_secs(5), + repo.save( + "sql", + "victim", + "small", + vec![3; 16], + ArtifactExpectation::Absent, + ), + ) + .await + .expect("a waiting large save does not pin the only pooled connection") + .unwrap(); + + gate.add_permits(1); + for holder in holders { + holder.await.unwrap().unwrap(); + } + tokio::time::timeout(Duration::from_secs(20), saver) + .await + .expect("the waiting save runs once a permit frees") + .unwrap() + .unwrap(); + assert_eq!(repo.large_transfers.available_permits(), 2); + } + + /// TC-2961: both limits compare the same stored bytes. A checkpoint + /// exactly at the write limit is readable under an equal read limit; a + /// read limit below it refuses it (startup refuses that configuration); + /// a read limit of 0 is off. + #[tokio::test] + async fn the_read_and_write_limits_compare_the_same_stored_bytes() { + let conn = Database::connect(ConnectOptions::new("sqlite::memory:".to_string())) + .await + .unwrap(); + Migrator::up(&conn, None).await.unwrap(); + let repo = SeaOrmDatabaseArtifactRepository::new(conn) + .with_max_write_bytes(4096) + .with_max_read_bytes(4096); + repo.save("sql", "s", "at", vec![1; 4096], ArtifactExpectation::Absent) + .await + .unwrap(); + let meta = repo.load_meta("sql", "s", "at").await.unwrap().unwrap(); + assert_eq!(meta.checkpoint_size_bytes, 4096); + let mut sink = MemorySink::default(); + repo.read_chunks("sql", "s", "at", &meta, &mut sink) + .await + .unwrap(); + + let lower = repo.clone().with_max_read_bytes(4095); + let mut sink = MemorySink::default(); + assert!(matches!( + lower.read_chunks("sql", "s", "at", &meta, &mut sink).await, + Err(DatabaseArtifactError::ReadLimitExceeded { .. }) + )); + + let off = repo.clone().with_max_read_bytes(0); + let mut sink = MemorySink::default(); + off.read_chunks("sql", "s", "at", &meta, &mut sink) + .await + .unwrap(); + assert_eq!(sink.checkpoint.len(), 4096); + } + #[tokio::test] async fn postgres_absent_create_refuses_an_existing_or_racing_row() { use sea_orm::{ConnectionTrait, DbBackend, Statement, TransactionTrait}; diff --git a/tinycloud-core/src/duckdb/service.rs b/tinycloud-core/src/duckdb/service.rs index d40bcd08..d78ed984 100644 --- a/tinycloud-core/src/duckdb/service.rs +++ b/tinycloud-core/src/duckdb/service.rs @@ -661,8 +661,13 @@ fn duckdb_wal_path(path: &Path) -> PathBuf { /// `DuckDbError` has no retryable-unavailable variant (adding one would reach /// every route mapping), so an exhausted `ArtifactChanged` stays `Internal`. +/// The artifact size limits answer 507, as for SQL (TC-2961). fn artifact_error_to_duckdb(err: DatabaseArtifactError) -> DuckDbError { - DuckDbError::Internal(err.to_string()) + match err { + DatabaseArtifactError::WriteLimitExceeded { .. } => DuckDbError::DatabaseSizeLimit, + DatabaseArtifactError::ReadLimitExceeded { .. } => DuckDbError::DatabaseTooLargeToLoad, + err => DuckDbError::Internal(err.to_string()), + } } #[cfg(test)] diff --git a/tinycloud-core/src/duckdb/types.rs b/tinycloud-core/src/duckdb/types.rs index 4ea020b6..9576802a 100644 --- a/tinycloud-core/src/duckdb/types.rs +++ b/tinycloud-core/src/duckdb/types.rs @@ -461,6 +461,14 @@ pub enum DuckDbError { QuotaExceeded, #[error("Operation would increase database storage")] StorageWouldGrow, + /// The database's whole checkpoint exceeds the artifact write limit + /// (TC-2961); it was not saved. + #[error("sql_database_size_limit: the write would exceed the database size limit; no change was applied")] + DatabaseSizeLimit, + /// The stored database exceeds the artifact read limit (TC-2961); it was + /// not loaded. + #[error("sql_database_size_limit: the database is larger than this node loads; no change was applied")] + DatabaseTooLargeToLoad, #[error("Invalid statement: {0}")] InvalidStatement(String), #[error("Schema error: {0}")] diff --git a/tinycloud-core/src/sql/service.rs b/tinycloud-core/src/sql/service.rs index 12c7bec2..8dfa9596 100644 --- a/tinycloud-core/src/sql/service.rs +++ b/tinycloud-core/src/sql/service.rs @@ -1071,8 +1071,8 @@ impl SqlService { return Err(SqlError::StorageWouldGrow); } } - let saved = match self - .bounded( + let (saved, transfer_started) = + crate::database_artifacts::track_transfer_start(self.bounded( (space, db_name), || SqlBusyStage::ArtifactTransfer, self.bounds.artifact_transfer, @@ -1083,9 +1083,9 @@ impl SqlService { payload, expected, ), - ) - .await - { + )) + .await; + let saved = match saved { Ok(Ok(saved)) => saved, Ok(Err(error)) => { // A stale actor's lineage conflict is recovered by the @@ -1098,7 +1098,11 @@ impl SqlService { return Err(error); } Err(error) => { - self.back_off_checkpoint(&key, &error); + // TC-2961: a save cut while still queued for its large-transfer + // slot sent nothing; it is busy, not a failed checkpoint. + if transfer_started { + self.back_off_checkpoint(&key, &error); + } return Err(error); } }; @@ -3177,6 +3181,129 @@ mod tests { ); } + /// Holds every load at its first part until the gate opens. + struct StalledSink { + entered: Arc, + gate: Arc, + } + + #[async_trait] + impl ArtifactSink for StalledSink { + async fn reset(&mut self) -> Result<(), DatabaseArtifactError> { + Ok(()) + } + async fn begin_part( + &mut self, + part: crate::database_artifacts::ArtifactPart, + ) -> Result<(), DatabaseArtifactError> { + if part == crate::database_artifacts::ArtifactPart::Checkpoint { + self.entered.fetch_add(1, Ordering::SeqCst); + drop(self.gate.acquire().await.unwrap()); + } + Ok(()) + } + async fn write(&mut self, _bytes: &[u8]) -> Result<(), DatabaseArtifactError> { + Ok(()) + } + async fn finish_part(&mut self) -> Result<(), DatabaseArtifactError> { + Ok(()) + } + } + + /// TC-2961 (Bobbie's F1): another tenant's two large loads hold both + /// large-transfer permits. This tenant's large checkpoint save queues for + /// a permit past `artifact_transfer`, sends nothing and is answered busy. + /// That is not a failed checkpoint: once the permits are free, its next + /// write goes through, with no back-off. + #[tokio::test] + async fn a_save_still_queued_for_a_permit_starts_no_back_off() { + let db = Database::connect(ConnectOptions::new("sqlite::memory:".to_string())) + .await + .unwrap(); + Migrator::up(&db, None).await.unwrap(); + let repo = SeaOrmDatabaseArtifactRepository::new(db); + let large = crate::database_artifacts::BASE_TIMEOUT_PAYLOAD_BYTES + 1; + for name in ["h1", "h2"] { + repo.save( + "sql", + "other-tenant", + name, + vec![1; large], + ArtifactExpectation::Absent, + ) + .await + .unwrap(); + } + let entered = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let holders: Vec<_> = ["h1", "h2"] + .into_iter() + .map(|name| { + let repo = repo.clone(); + let mut sink = StalledSink { + entered: entered.clone(), + gate: gate.clone(), + }; + tokio::spawn(async move { + let meta = repo + .load_meta("sql", "other-tenant", name) + .await + .unwrap() + .unwrap(); + repo.read_chunks("sql", "other-tenant", name, &meta, &mut sink) + .await + }) + }) + .collect(); + tokio::time::timeout(Duration::from_secs(10), async { + while entered.load(Ordering::SeqCst) < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the holders start"); + + let cache = TempDir::new().unwrap(); + let service = SqlService::new( + cache.path().to_string_lossy().to_string(), + u64::MAX, + Arc::new(repo.clone()), + ) + .with_wait_bounds(SqlWaitBounds { + artifact_transfer: Duration::from_millis(500), + ..SqlWaitBounds::default() + }); + let space = test_space_id("queued-save"); + let write = |table: &str, value: &str| SqlRequest::Execute { + schema: Some(vec![format!("CREATE TABLE IF NOT EXISTS {table} (v)")]), + sql: format!("INSERT INTO {table} VALUES ({value})"), + params: Vec::new(), + }; + let admin = || "tinycloud.sql/admin".to_string(); + let queued = service + .execute( + &space, + "main", + write("b", "zeroblob(9437184)"), + None, + admin(), + ) + .await; + assert!( + matches!(queued, Err(SqlError::Busy(SqlBusyStage::ArtifactTransfer))), + "{queued:?}" + ); + + gate.add_permits(1); + for holder in holders { + holder.await.unwrap().unwrap(); + } + let next = service + .execute(&space, "main", write("s", "1"), None, admin()) + .await; + assert!(next.is_ok(), "no back-off: {next:?}"); + } + /// Fails the next `failures` repository calls with `error`, then /// delegates. struct FlakyArtifactRepository { diff --git a/tinycloud-core/tests/artifact_bounded_read.rs b/tinycloud-core/tests/artifact_bounded_read.rs index 6d5ae21d..dd376aea 100644 --- a/tinycloud-core/tests/artifact_bounded_read.rs +++ b/tinycloud-core/tests/artifact_bounded_read.rs @@ -93,7 +93,8 @@ async fn a_256_mib_artifact_hydrates_with_a_bounded_heap() { .await .unwrap(); - let repo = SeaOrmDatabaseArtifactRepository::new(conn); + // The TC-2961 read cap is off to measure the streaming path. + let repo = SeaOrmDatabaseArtifactRepository::new(conn).with_max_read_bytes(0); let cache = tempfile::tempdir().unwrap(); let mut sink = CacheFileSink::new( "sql", diff --git a/tinycloud-node-server/src/config.rs b/tinycloud-node-server/src/config.rs index 6e55df7a..0c435115 100644 --- a/tinycloud-node-server/src/config.rs +++ b/tinycloud-node-server/src/config.rs @@ -1664,7 +1664,8 @@ pub struct SqlStorageConfig { /// The largest stored SQL or DuckDB checkpoint or WAL delta the node /// loads (TC-2961). A larger database is refused before any of it is /// read (`507 sql_database_size_limit`) instead of being loaded whole. - /// On by default, 64 MiB; 0 turns it off. + /// On by default, 64 MiB; 0 turns it off. Otherwise `validate` refuses + /// it below `max_artifact_write_bytes`, or with that off. #[serde(default = "default_sql_max_artifact_read_bytes")] pub max_artifact_read_bytes: ByteUnit, } @@ -1724,6 +1725,19 @@ impl SqlStorageConfig { hydration_max: Duration::from_millis(self.hydration_max_ms), } } + + /// Refuse an artifact read limit below the write limit (TC-2961): the + /// databases between the two would be saved, then refused on every load. + pub fn validate(&self) -> Result<(), &'static str> { + let read = self.max_artifact_read_bytes.as_u64(); + let write = self.max_artifact_write_bytes.as_u64(); + if read != 0 && (write == 0 || read < write) { + return Err( + "storage.sql.max_artifact_read_bytes must be 0 (off) or at least storage.sql.max_artifact_write_bytes (and that must not be 0)", + ); + } + Ok(()) + } } impl Default for SqlStorageConfig { @@ -2394,6 +2408,31 @@ mod tests { } } + /// TC-2961: a read limit below the write limit would make databases + /// between the two writable and then unloadable, so the node refuses to + /// start with it. 0 turns the read limit off. + #[test] + fn startup_refuses_an_artifact_read_limit_below_the_write_limit() { + let mib = |n: u64| ByteUnit::Byte(n * 1024 * 1024); + for (write, read, ok) in [ + (mib(64), mib(64), true), + (mib(64), mib(128), true), + (mib(64), ByteUnit::Byte(0), true), + (ByteUnit::Byte(0), ByteUnit::Byte(0), true), + (mib(64), ByteUnit::Byte(64 * 1024 * 1024 - 1), false), + (ByteUnit::Byte(0), mib(64), false), + ] { + let mut config = Config::default(); + config.storage.sql.max_artifact_write_bytes = write; + config.storage.sql.max_artifact_read_bytes = read; + assert_eq!( + crate::resolve_runtime_config(&config).is_ok(), + ok, + "write {write:?} read {read:?}" + ); + } + } + /// TC-2954: a statement timeout above PostgreSQL's largest is refused at /// startup instead of failing every connection after the old node has /// stopped. diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 5d11c67a..da1f4364 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -594,6 +594,11 @@ pub fn resolve_runtime_config(tinycloud_config: &Config) -> Result { let mut resolved = tinycloud_config.clone(); resolved.hooks.validate().map_err(anyhow::Error::msg)?; resolved.database.validate().map_err(anyhow::Error::msg)?; + resolved + .storage + .sql + .validate() + .map_err(anyhow::Error::msg)?; resolved .epoch_head_backfill .validate() diff --git a/tinycloud-node-server/src/routes/mod.rs b/tinycloud-node-server/src/routes/mod.rs index e478c38d..81c26062 100644 --- a/tinycloud-node-server/src/routes/mod.rs +++ b/tinycloud-node-server/src/routes/mod.rs @@ -4143,6 +4143,7 @@ fn duckdb_error_to_status(err: &DuckDbError) -> Status { DuckDbError::ResponseTooLarge(..) => Status::new(413), DuckDbError::QuotaExceeded => Status::new(429), DuckDbError::StorageWouldGrow => Status::new(402), + DuckDbError::DatabaseSizeLimit | DuckDbError::DatabaseTooLargeToLoad => Status::new(507), DuckDbError::IngestError(_) => Status::InternalServerError, DuckDbError::ExportError(_) => Status::InternalServerError, DuckDbError::ImportError(_) => Status::InternalServerError, @@ -8381,6 +8382,26 @@ mod tests { .starts_with("sql_commit_failed:")); } + /// TC-2961: a database too large to load answers 507 with the size-limit + /// code, for SQL and DuckDB alike, never 500. + #[test] + fn a_database_too_large_to_load_is_507() { + assert_eq!( + sql_error_to_status(&SqlError::DatabaseTooLargeToLoad), + Status::new(507) + ); + let body = SqlError::DatabaseTooLargeToLoad.to_string(); + assert!(body.starts_with("sql_database_size_limit:"), "{body}"); + #[cfg(feature = "duckdb")] + for error in [ + DuckDbError::DatabaseSizeLimit, + DuckDbError::DatabaseTooLargeToLoad, + ] { + assert_eq!(duckdb_error_to_status(&error), Status::new(507)); + assert!(error.to_string().starts_with("sql_database_size_limit:")); + } + } + #[tokio::test] async fn sql_request_is_write_classifies_variants() { use tinycloud_core::sql::types::SqlStatement;