From 9592b84824d43e9534098f70984b7e98c9fb4902 Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:36:15 +0000 Subject: [PATCH 1/4] fix: end PostgreSQL transactions left idle for 15 s (TC-2954) Every connection in the node's PostgreSQL pool sets idle_in_transaction_session_timeout once at connect (database.idle_in_transaction_timeout_ms, default 15000, 0 off), so a stalled or leaked transaction cannot keep holding its locks and pooled connection; the pool replaces the ended session. --- CHANGELOG.md | 1 + README.md | 1 + tinycloud-node-server/src/config.rs | 24 +++++++ tinycloud-node-server/src/lib.rs | 8 ++- .../tests/postgres_pool_timeouts.rs | 62 +++++++++++++++++++ 5 files changed, 95 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 640e21d09..492c25323 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] ### Changes +- PostgreSQL: every pooled connection sets `idle_in_transaction_session_timeout` (`database.idle_in_transaction_timeout_ms`, env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`, default 15000 = 15 s, `0` off), so the server ends a transaction left idle that long and the pool replaces the session. A protected legacy meeting write whose upload takes longer than the timeout now fails. Details: TC-2954. - SQLite stamp replay deadlines now use the same timestamp encoding as cleanup (TC-2920). - PostgreSQL database artifacts use uncompressed external storage for new writes to avoid repeated decompression during chunked reads (TC-2917). Existing large artifacts require an operator re-toast; verify both storage settings after rollout and follow `scripts/tc2917-README.md` if lock contention deferred the migration. - Encryption: v2 column format with key ids, per-purpose keys and recorded format metadata. Optional strict reads are off by default. Details: TC-2516, TC-2517. diff --git a/README.md b/README.md index f5592fad2..a26cfaddb 100644 --- a/README.md +++ b/README.md @@ -303,6 +303,7 @@ These bound how long one PostgreSQL round trip can block a request (TC-1054). Ea | database.statement_timeout_mode | `session` | How PostgreSQL's `statement_timeout` is applied: `session` sets it once on each new connection (no per-request cost; covers every statement); `local` sets it with `SET LOCAL` in each explicit transaction (safe behind a transaction-pooling proxy, but one more round trip per transaction, and autocommit statements are not covered); `off` disables it. | | database.statement_timeout_ms | `30000` | `statement_timeout` for every statement. `0` disables it. | | database.bulk_statement_timeout_ms | `120000` | `statement_timeout` for statements on a whole SQL or DuckDB artifact larger than 8 MiB (or of unknown size): hydration loads and checkpoint saves, whose server-side work on a ~100 MiB value can outlast the base limit. PostgreSQL does not cancel a statement while it writes a row to the client, so the download itself is not limited by either value. | +| database.idle_in_transaction_timeout_ms | `15000` | PostgreSQL `idle_in_transaction_session_timeout`, set once on each new PostgreSQL connection (env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`): the server ends a session left idle inside a transaction this long, so a stalled or leaked transaction cannot keep holding its locks and pooled connection; the pool replaces the session. A protected legacy meeting write holds its transaction across its upload, so one whose upload takes longer than this fails. `0` disables it. SQLite is unaffected. | | database.retry_window_ms | `10000` | A read (or `BEGIN`) whose connection is lost is retried once only if that attempt failed within this window. A connection found dead at once is retried; one whose loss took a keepalive expiry to detect is not, so a lost connection costs a read at most one detection plus this window (about 40 s) instead of two detections (about 60 s). This bounds the retry, not the request or the per-database lock hold: acquisition waits, legitimate transfers and queueing behind the lock add to it, and a general deadline is later TC-1054 work. | Schema migrations always run with no statement timeout. On PostgreSQL they wait at most 30 s for any lock (`lock_timeout`), so a session holding a conflicting table lock makes the node fail to start with an error naming the lock wait instead of hanging; the migrations roll back together and the next start applies them (TC-1176). Customer SQL (`tinycloud.sql/*`) runs in the node's local SQLite engine, never on PostgreSQL, so `statement_timeout` does not limit it. diff --git a/tinycloud-node-server/src/config.rs b/tinycloud-node-server/src/config.rs index 3fa9efec0..92d053b7e 100644 --- a/tinycloud-node-server/src/config.rs +++ b/tinycloud-node-server/src/config.rs @@ -1929,6 +1929,14 @@ pub struct DatabaseConfig { /// writing a row to the client, so the download itself is not limited. #[serde(default = "default_database_bulk_statement_timeout_ms")] pub bulk_statement_timeout_ms: u64, + /// PostgreSQL `idle_in_transaction_session_timeout`, in milliseconds, set + /// once on every pooled connection (TC-2954): the server ends a session + /// that sits idle inside a transaction this long, so a leaked or stalled + /// transaction cannot hold its locks and pooled connection. A protected + /// legacy meeting write holds its transaction across its upload, so one + /// whose upload takes longer fails. Zero disables it. + #[serde(default = "default_database_idle_in_transaction_timeout_ms")] + pub idle_in_transaction_timeout_ms: u64, /// Enable TCP keepalive and `TCP_USER_TIMEOUT` on PostgreSQL connection /// sockets, so a connection the network silently drops fails within tens /// of seconds instead of the kernel's ~15 min retransmission timeout @@ -1995,6 +2003,10 @@ fn default_database_bulk_statement_timeout_ms() -> u64 { 120_000 } +fn default_database_idle_in_transaction_timeout_ms() -> u64 { + 15_000 +} + fn default_database_tcp_keepalive_idle_secs() -> u64 { 15 } @@ -2029,6 +2041,7 @@ impl Default for DatabaseConfig { statement_timeout_mode: StatementTimeoutMode::default(), statement_timeout_ms: default_database_statement_timeout_ms(), bulk_statement_timeout_ms: default_database_bulk_statement_timeout_ms(), + idle_in_transaction_timeout_ms: default_database_idle_in_transaction_timeout_ms(), tcp_keepalive: true, tcp_keepalive_idle_secs: default_database_tcp_keepalive_idle_secs(), tcp_keepalive_interval_secs: default_database_tcp_keepalive_interval_secs(), @@ -2060,6 +2073,17 @@ impl DatabaseConfig { Ok(()) } + /// The statement that sets the pool's idle-in-transaction timeout on a new + /// connection, or `None` when it is off. + pub fn idle_in_transaction_statement(&self) -> Option { + (self.idle_in_transaction_timeout_ms > 0).then(|| { + format!( + "SET idle_in_transaction_session_timeout = {}", + self.idle_in_transaction_timeout_ms + ) + }) + } + /// The pool's PostgreSQL `statement_timeout` policy. pub fn statement_timeouts(&self) -> StatementTimeouts { StatementTimeouts { diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index f7f05cec0..b483c0aee 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -516,6 +516,7 @@ pub async fn postgres_pool_with_recovery( ); let statement_timeouts = database.statement_timeouts(); let session_statement = statement_timeouts.session_statement(); + let idle_in_transaction_statement = database.idle_in_transaction_statement(); let pool = recovery .install( connect_opts @@ -524,9 +525,13 @@ pub async fn postgres_pool_with_recovery( ) .after_connect(move |conn, _meta| { let session_statement = session_statement.clone(); + let idle_in_transaction_statement = idle_in_transaction_statement.clone(); Box::pin(async move { POSTGRES_CONNECTIONS_OPENED.fetch_add(1, Ordering::Relaxed); - if let Some(statement) = session_statement { + for statement in [session_statement, idle_in_transaction_statement] + .into_iter() + .flatten() + { conn.execute(statement.as_str()).await?; } Ok(()) @@ -541,6 +546,7 @@ pub async fn postgres_pool_with_recovery( statement_timeout_mode = ?statement_timeouts.mode, statement_timeout_ms = statement_timeouts.base.as_millis() as u64, bulk_statement_timeout_ms = statement_timeouts.bulk.as_millis() as u64, + idle_in_transaction_timeout_ms = database.idle_in_transaction_timeout_ms, retry_window_ms = database.retry_window_ms, tcp_keepalive = keepalive.is_some(), tcp_keepalive_idle_secs = keepalive.map(|k| k.idle.as_secs()), diff --git a/tinycloud-node-server/tests/postgres_pool_timeouts.rs b/tinycloud-node-server/tests/postgres_pool_timeouts.rs index 1792d90d1..eebb4f6ca 100644 --- a/tinycloud-node-server/tests/postgres_pool_timeouts.rs +++ b/tinycloud-node-server/tests/postgres_pool_timeouts.rs @@ -179,6 +179,15 @@ async fn show_statement_timeout(conn: &C) -> String { .unwrap() } +async fn show(conn: &C, setting: &str) -> String { + conn.query_one(statement(&format!("SHOW {setting}"))) + .await + .unwrap() + .unwrap() + .try_get_by_index::(0) + .unwrap() +} + fn test_space(name: &str) -> SpaceId { let jwk = JWK::generate_ed25519().unwrap(); let did: DIDBuf = DID_METHODS.generate(&jwk, "key").unwrap(); @@ -434,6 +443,59 @@ async fn postgres_statement_timeout_modes_scopes_and_migrations() { schema.drop().await; } +/// TC-2954: every pooled connection carries an idle-in-transaction timeout, +/// 15 s by default, so the server ends a transaction left idle that long and +/// a fresh connection serves the next request; 0 turns it off. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn postgres_idle_in_transaction_timeout_ends_an_abandoned_transaction() { + let Some(url) = + postgres_url("postgres_idle_in_transaction_timeout_ends_an_abandoned_transaction") + else { + return; + }; + let schema = Schema::new(&url, "idletx").await; + let setting = "idle_in_transaction_session_timeout"; + + let default = pool(schema.url(&url, None), |_| {}).await; + assert_eq!(show(&default, setting).await, "15s"); + let off = pool(schema.url(&url, None), |config| { + config.database.idle_in_transaction_timeout_ms = 0; + }) + .await; + assert_eq!(show(&off, setting).await, "0"); + + let short = pool(schema.url(&url, None), |config| { + config.database.idle_in_transaction_timeout_ms = 300; + }) + .await; + let transaction = short.begin().await.unwrap(); + let abandoned = transaction + .query_one(statement("SELECT pg_backend_pid()")) + .await + .unwrap() + .unwrap() + .try_get_by_index::(0) + .unwrap(); + tokio::time::sleep(Duration::from_millis(1_000)).await; + transaction + .execute(statement("SELECT 1")) + .await + .expect_err("the server ended the idle transaction"); + drop(transaction); + let next = short + .query_one(statement("SELECT pg_backend_pid()")) + .await + .expect("a fresh connection serves the next request") + .unwrap() + .try_get_by_index::(0) + .unwrap(); + assert_ne!(next, abandoned, "the ended session is not reused"); + for conn in [default, off, short] { + conn.sea_orm_connection().clone().close().await.unwrap(); + } + schema.drop().await; +} + /// A statement timeout on the SQL persistence path is a bounded, retryable /// `Unavailable` (503): the database's operation lock is released, the /// next request on it succeeds, and other databases are unaffected. A full From 940e12418ca651dd7b69a9c4cab3e7e9d489a6b2 Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:45:22 +0000 Subject: [PATCH 2/4] fix: default the idle-in-transaction timeout to 300 s and refuse values within the 240 s upload hold (TC-2954) --- CHANGELOG.md | 2 +- README.md | 2 +- tinycloud-core/src/db.rs | 4 +- tinycloud-node-server/src/config.rs | 41 ++++++++++++++++--- .../tests/postgres_pool_timeouts.rs | 5 ++- 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 492c25323..80bee5eb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [Unreleased] ### Changes -- PostgreSQL: every pooled connection sets `idle_in_transaction_session_timeout` (`database.idle_in_transaction_timeout_ms`, env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`, default 15000 = 15 s, `0` off), so the server ends a transaction left idle that long and the pool replaces the session. A protected legacy meeting write whose upload takes longer than the timeout now fails. Details: TC-2954. +- PostgreSQL: every pooled connection sets `idle_in_transaction_session_timeout` (`database.idle_in_transaction_timeout_ms`, env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`, default 300000 = 5 min, `0` off), so the server ends a transaction left idle that long and the pool replaces the session. A non-zero value of 240 s or less is refused at startup: a protected legacy meeting write legitimately holds its transaction across an upload of up to 240 s. Details: TC-2954. - SQLite stamp replay deadlines now use the same timestamp encoding as cleanup (TC-2920). - PostgreSQL database artifacts use uncompressed external storage for new writes to avoid repeated decompression during chunked reads (TC-2917). Existing large artifacts require an operator re-toast; verify both storage settings after rollout and follow `scripts/tc2917-README.md` if lock contention deferred the migration. - Encryption: v2 column format with key ids, per-purpose keys and recorded format metadata. Optional strict reads are off by default. Details: TC-2516, TC-2517. diff --git a/README.md b/README.md index a26cfaddb..8e9c84220 100644 --- a/README.md +++ b/README.md @@ -303,7 +303,7 @@ These bound how long one PostgreSQL round trip can block a request (TC-1054). Ea | database.statement_timeout_mode | `session` | How PostgreSQL's `statement_timeout` is applied: `session` sets it once on each new connection (no per-request cost; covers every statement); `local` sets it with `SET LOCAL` in each explicit transaction (safe behind a transaction-pooling proxy, but one more round trip per transaction, and autocommit statements are not covered); `off` disables it. | | database.statement_timeout_ms | `30000` | `statement_timeout` for every statement. `0` disables it. | | database.bulk_statement_timeout_ms | `120000` | `statement_timeout` for statements on a whole SQL or DuckDB artifact larger than 8 MiB (or of unknown size): hydration loads and checkpoint saves, whose server-side work on a ~100 MiB value can outlast the base limit. PostgreSQL does not cancel a statement while it writes a row to the client, so the download itself is not limited by either value. | -| database.idle_in_transaction_timeout_ms | `15000` | PostgreSQL `idle_in_transaction_session_timeout`, set once on each new PostgreSQL connection (env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`): the server ends a session left idle inside a transaction this long, so a stalled or leaked transaction cannot keep holding its locks and pooled connection; the pool replaces the session. A protected legacy meeting write holds its transaction across its upload, so one whose upload takes longer than this fails. `0` disables it. SQLite is unaffected. | +| database.idle_in_transaction_timeout_ms | `300000` | PostgreSQL `idle_in_transaction_session_timeout`, set once on each new PostgreSQL connection (env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`): the server ends a session left idle inside a transaction this long, so a stalled or leaked transaction cannot keep holding its locks and pooled connection; the pool replaces the session. It must exceed the 240 s a protected legacy meeting write holds its transaction across its upload; the node refuses to start with a non-zero value of 240 s or less. `0` disables it. SQLite is unaffected. | | database.retry_window_ms | `10000` | A read (or `BEGIN`) whose connection is lost is retried once only if that attempt failed within this window. A connection found dead at once is retried; one whose loss took a keepalive expiry to detect is not, so a lost connection costs a read at most one detection plus this window (about 40 s) instead of two detections (about 60 s). This bounds the retry, not the request or the per-database lock hold: acquisition waits, legitimate transfers and queueing behind the lock add to it, and a general deadline is later TC-1054 work. | Schema migrations always run with no statement timeout. On PostgreSQL they wait at most 30 s for any lock (`lock_timeout`), so a session holding a conflicting table lock makes the node fail to start with an error naming the lock wait instead of hanging; the migrations roll back together and the next start applies them (TC-1176). Customer SQL (`tinycloud.sql/*`) runs in the node's local SQLite engine, never on PostgreSQL, so `statement_timeout` does not limit it. diff --git a/tinycloud-core/src/db.rs b/tinycloud-core/src/db.rs index 986fdb69c..65237b79d 100644 --- a/tinycloud-core/src/db.rs +++ b/tinycloud-core/src/db.rs @@ -61,7 +61,9 @@ pub const KV_MAX_MUTATIONS_PER_INVOCATION: usize = 4096; /// One invocation's uploads finish within 240 seconds of preflight, before /// the five-minute orphan age. A live same-hash retry defers an older crash /// stage until that retry finishes; repeated overlaps can extend its charge. -const KV_UPLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(240); +/// A protected legacy write holds its transaction across this upload, so the +/// pool's idle-in-transaction timeout must exceed it (TC-2954). +pub const KV_UPLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(240); const _: () = assert!(KV_UPLOAD_TIMEOUT.as_secs() < crate::kv_orphans::ORPHAN_GRACE.whole_seconds() as u64); diff --git a/tinycloud-node-server/src/config.rs b/tinycloud-node-server/src/config.rs index 92d053b7e..1dc84014d 100644 --- a/tinycloud-node-server/src/config.rs +++ b/tinycloud-node-server/src/config.rs @@ -1932,9 +1932,10 @@ pub struct DatabaseConfig { /// PostgreSQL `idle_in_transaction_session_timeout`, in milliseconds, set /// once on every pooled connection (TC-2954): the server ends a session /// that sits idle inside a transaction this long, so a leaked or stalled - /// transaction cannot hold its locks and pooled connection. A protected - /// legacy meeting write holds its transaction across its upload, so one - /// whose upload takes longer fails. Zero disables it. + /// transaction cannot hold its locks and pooled connection indefinitely. + /// It must exceed the longest legitimate hold, a protected legacy meeting + /// write's upload (`KV_UPLOAD_TIMEOUT`, 240 s); `validate` refuses less. + /// Zero disables it. #[serde(default = "default_database_idle_in_transaction_timeout_ms")] pub idle_in_transaction_timeout_ms: u64, /// Enable TCP keepalive and `TCP_USER_TIMEOUT` on PostgreSQL connection @@ -2004,7 +2005,7 @@ fn default_database_bulk_statement_timeout_ms() -> u64 { } fn default_database_idle_in_transaction_timeout_ms() -> u64 { - 15_000 + 300_000 } fn default_database_tcp_keepalive_idle_secs() -> u64 { @@ -2065,11 +2066,21 @@ pub struct DidResolutionConfig { impl DatabaseConfig { /// Reject settings that would make the node unusable (TC-1317): a zero - /// read-audit wait would answer every authorized read `read_audit_busy`. + /// read-audit wait would answer every authorized read `read_audit_busy`, + /// and an idle-in-transaction timeout within the legacy upload hold would + /// fail legitimate writes (TC-2954). pub fn validate(&self) -> Result<(), &'static str> { if self.read_audit_wait_ms == 0 { return Err("database.read_audit_wait_ms must be greater than zero"); } + if self.idle_in_transaction_timeout_ms != 0 + && u128::from(self.idle_in_transaction_timeout_ms) + <= tinycloud_core::db::KV_UPLOAD_TIMEOUT.as_millis() + { + return Err( + "database.idle_in_transaction_timeout_ms must be 0 (off) or exceed the 240 s legacy upload hold", + ); + } Ok(()) } @@ -2317,6 +2328,26 @@ mod tests { } } + /// TC-2954: the idle-in-transaction timeout is on by default (300 s) and, + /// when on, must outlast the 240 s a protected legacy write holds its + /// transaction. + #[test] + fn tc2954_startup_rejects_an_idle_in_transaction_timeout_within_the_upload_hold() { + assert_eq!( + Config::default().database.idle_in_transaction_timeout_ms, + 300_000 + ); + for (timeout_ms, ok) in [(0, true), (1, false), (240_000, false), (240_001, true)] { + let mut config = Config::default(); + config.database.idle_in_transaction_timeout_ms = timeout_ms; + assert_eq!( + crate::resolve_runtime_config(&config).is_ok(), + ok, + "{timeout_ms}" + ); + } + } + /// TC-1491: off by default, 60 s / 8 MiB for every tier, and nothing is /// validated while the feature is off. #[test] diff --git a/tinycloud-node-server/tests/postgres_pool_timeouts.rs b/tinycloud-node-server/tests/postgres_pool_timeouts.rs index eebb4f6ca..59659cb33 100644 --- a/tinycloud-node-server/tests/postgres_pool_timeouts.rs +++ b/tinycloud-node-server/tests/postgres_pool_timeouts.rs @@ -444,7 +444,7 @@ async fn postgres_statement_timeout_modes_scopes_and_migrations() { } /// TC-2954: every pooled connection carries an idle-in-transaction timeout, -/// 15 s by default, so the server ends a transaction left idle that long and +/// 300 s by default, so the server ends a transaction left idle that long and /// a fresh connection serves the next request; 0 turns it off. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn postgres_idle_in_transaction_timeout_ends_an_abandoned_transaction() { @@ -457,13 +457,14 @@ async fn postgres_idle_in_transaction_timeout_ends_an_abandoned_transaction() { let setting = "idle_in_transaction_session_timeout"; let default = pool(schema.url(&url, None), |_| {}).await; - assert_eq!(show(&default, setting).await, "15s"); + assert_eq!(show(&default, setting).await, "5min"); let off = pool(schema.url(&url, None), |config| { config.database.idle_in_transaction_timeout_ms = 0; }) .await; assert_eq!(show(&off, setting).await, "0"); + // Below the legacy upload hold `validate` refuses; the pool itself takes it. let short = pool(schema.url(&url, None), |config| { config.database.idle_in_transaction_timeout_ms = 300; }) From 4b3a6448c4497b4b5b99142ea1205f8dda5989e5 Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:13:53 +0000 Subject: [PATCH 3/4] test: an ended idle transaction is one prompt connection-class error and one replacement connection (TC-2954) --- .../tests/postgres_pool_timeouts.rs | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tinycloud-node-server/tests/postgres_pool_timeouts.rs b/tinycloud-node-server/tests/postgres_pool_timeouts.rs index 59659cb33..26bf8d2d0 100644 --- a/tinycloud-node-server/tests/postgres_pool_timeouts.rs +++ b/tinycloud-node-server/tests/postgres_pool_timeouts.rs @@ -478,10 +478,21 @@ async fn postgres_idle_in_transaction_timeout_ends_an_abandoned_transaction() { .try_get_by_index::(0) .unwrap(); tokio::time::sleep(Duration::from_millis(1_000)).await; - transaction + // The request whose transaction was ended gets one prompt error of the + // connection class: the routes answer it 503 unavailable_retryable, or + // 500 outcome_unknown once a write began (TC-1054). Statements inside a + // transaction are never retried, so nothing reconnects in a loop. + let started = Instant::now(); + let error = transaction .execute(statement("SELECT 1")) .await .expect_err("the server ended the idle transaction"); + assert!( + started.elapsed() < Duration::from_millis(500), + "{:?}", + started.elapsed() + ); + assert!(is_transient_error(&error), "{error:?}"); drop(transaction); let next = short .query_one(statement("SELECT pg_backend_pid()")) @@ -491,6 +502,15 @@ async fn postgres_idle_in_transaction_timeout_ends_an_abandoned_transaction() { .try_get_by_index::(0) .unwrap(); assert_ne!(next, abandoned, "the ended session is not reused"); + // The ended session is closed, not kept: one replacement serves the pool. + let pool = short.sea_orm_connection().get_postgres_connection_pool(); + tokio::time::timeout(Duration::from_secs(5), async { + while pool.size() != 1 { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the pool holds exactly the replacement connection"); for conn in [default, off, short] { conn.sea_orm_connection().clone().close().await.unwrap(); } From 1ec889e13e68b123b50f28a9955473097ca07fb5 Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:06:26 +0000 Subject: [PATCH 4/4] fix(TC-2954): bound PostgreSQL timeouts at INT_MAX and skip the session idle timeout in local mode --- CHANGELOG.md | 2 +- README.md | 6 +- tinycloud-node-server/src/config.rs | 90 +++++++++++++++++++++++++---- tinycloud-node-server/src/lib.rs | 8 ++- 4 files changed, 90 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80bee5eb3..3f6016236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [Unreleased] ### Changes -- PostgreSQL: every pooled connection sets `idle_in_transaction_session_timeout` (`database.idle_in_transaction_timeout_ms`, env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`, default 300000 = 5 min, `0` off), so the server ends a transaction left idle that long and the pool replaces the session. A non-zero value of 240 s or less is refused at startup: a protected legacy meeting write legitimately holds its transaction across an upload of up to 240 s. Details: TC-2954. +- PostgreSQL: every pooled connection sets `idle_in_transaction_session_timeout` (`database.idle_in_transaction_timeout_ms`, env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`, default 300000 = 5 min, `0` off), so the server ends a transaction left idle that long and the pool replaces the session. A non-zero value of 240 s or less is refused at startup: a protected legacy meeting write legitimately holds its transaction across an upload of up to 240 s. It is not set in `statement_timeout_mode = "local"` (transaction-pooling proxies). `database.idle_in_transaction_timeout_ms`, `statement_timeout_ms` and `bulk_statement_timeout_ms` above 2147483647 (PostgreSQL's largest) are now refused at startup instead of failing every connection. Details: TC-2954. - SQLite stamp replay deadlines now use the same timestamp encoding as cleanup (TC-2920). - PostgreSQL database artifacts use uncompressed external storage for new writes to avoid repeated decompression during chunked reads (TC-2917). Existing large artifacts require an operator re-toast; verify both storage settings after rollout and follow `scripts/tc2917-README.md` if lock contention deferred the migration. - Encryption: v2 column format with key ids, per-purpose keys and recorded format metadata. Optional strict reads are off by default. Details: TC-2516, TC-2517. diff --git a/README.md b/README.md index 8e9c84220..5b50701df 100644 --- a/README.md +++ b/README.md @@ -301,9 +301,9 @@ These bound how long one PostgreSQL round trip can block a request (TC-1054). Ea | database.tcp_keepalive_retries | `3` | Unanswered probes before the connection is dropped (15 + 3 × 5 = 30 s). | | database.tcp_user_timeout_ms | `30000` | `TCP_USER_TIMEOUT`: how long sent data may go unacknowledged before the connection is dropped. `0` keeps the kernel default. | | database.statement_timeout_mode | `session` | How PostgreSQL's `statement_timeout` is applied: `session` sets it once on each new connection (no per-request cost; covers every statement); `local` sets it with `SET LOCAL` in each explicit transaction (safe behind a transaction-pooling proxy, but one more round trip per transaction, and autocommit statements are not covered); `off` disables it. | -| database.statement_timeout_ms | `30000` | `statement_timeout` for every statement. `0` disables it. | -| database.bulk_statement_timeout_ms | `120000` | `statement_timeout` for statements on a whole SQL or DuckDB artifact larger than 8 MiB (or of unknown size): hydration loads and checkpoint saves, whose server-side work on a ~100 MiB value can outlast the base limit. PostgreSQL does not cancel a statement while it writes a row to the client, so the download itself is not limited by either value. | -| database.idle_in_transaction_timeout_ms | `300000` | PostgreSQL `idle_in_transaction_session_timeout`, set once on each new PostgreSQL connection (env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`): the server ends a session left idle inside a transaction this long, so a stalled or leaked transaction cannot keep holding its locks and pooled connection; the pool replaces the session. It must exceed the 240 s a protected legacy meeting write holds its transaction across its upload; the node refuses to start with a non-zero value of 240 s or less. `0` disables it. SQLite is unaffected. | +| database.statement_timeout_ms | `30000` | `statement_timeout` for every statement. `0` disables it. At most 2147483647 (PostgreSQL's largest); the node refuses to start with more. | +| database.bulk_statement_timeout_ms | `120000` | `statement_timeout` for statements on a whole SQL or DuckDB artifact larger than 8 MiB (or of unknown size): hydration loads and checkpoint saves, whose server-side work on a ~100 MiB value can outlast the base limit. PostgreSQL does not cancel a statement while it writes a row to the client, so the download itself is not limited by either value. At most 2147483647; the node refuses to start with more. | +| database.idle_in_transaction_timeout_ms | `300000` | PostgreSQL `idle_in_transaction_session_timeout`, set once on each new PostgreSQL connection (env `TINYCLOUD_DATABASE__IDLE_IN_TRANSACTION_TIMEOUT_MS`): the server ends a session left idle inside a transaction this long, so a stalled or leaked transaction cannot keep holding its locks and pooled connection; the pool replaces the session. It must exceed the 240 s a protected legacy meeting write holds its transaction across its upload; the node refuses to start with a non-zero value of 240 s or less, or more than 2147483647. `0` disables it. Not set when `statement_timeout_mode` is `local`: behind a transaction-pooling proxy a session setting would reach the proxy's other clients. SQLite is unaffected; the operator tool `tinycloud-sql-identity` opens its own pool without it. | | database.retry_window_ms | `10000` | A read (or `BEGIN`) whose connection is lost is retried once only if that attempt failed within this window. A connection found dead at once is retried; one whose loss took a keepalive expiry to detect is not, so a lost connection costs a read at most one detection plus this window (about 40 s) instead of two detections (about 60 s). This bounds the retry, not the request or the per-database lock hold: acquisition waits, legitimate transfers and queueing behind the lock add to it, and a general deadline is later TC-1054 work. | Schema migrations always run with no statement timeout. On PostgreSQL they wait at most 30 s for any lock (`lock_timeout`), so a session holding a conflicting table lock makes the node fail to start with an error naming the lock wait instead of hanging; the migrations roll back together and the next start applies them (TC-1176). Customer SQL (`tinycloud.sql/*`) runs in the node's local SQLite engine, never on PostgreSQL, so `statement_timeout` does not limit it. diff --git a/tinycloud-node-server/src/config.rs b/tinycloud-node-server/src/config.rs index 1dc84014d..f7fc8bd4b 100644 --- a/tinycloud-node-server/src/config.rs +++ b/tinycloud-node-server/src/config.rs @@ -1935,7 +1935,10 @@ pub struct DatabaseConfig { /// transaction cannot hold its locks and pooled connection indefinitely. /// It must exceed the longest legitimate hold, a protected legacy meeting /// write's upload (`KV_UPLOAD_TIMEOUT`, 240 s); `validate` refuses less. - /// Zero disables it. + /// Zero disables it. Not set in `statement_timeout_mode = "local"`: a + /// session `SET` behind a transaction-pooling proxy would land on + /// whichever server connection served it and reach the proxy's other + /// clients. #[serde(default = "default_database_idle_in_transaction_timeout_ms")] pub idle_in_transaction_timeout_ms: u64, /// Enable TCP keepalive and `TCP_USER_TIMEOUT` on PostgreSQL connection @@ -2008,6 +2011,9 @@ fn default_database_idle_in_transaction_timeout_ms() -> u64 { 300_000 } +/// PostgreSQL's largest millisecond timeout setting (`INT_MAX`). +const POSTGRES_MAX_TIMEOUT_MS: u64 = i32::MAX as u64; + fn default_database_tcp_keepalive_idle_secs() -> u64 { 15 } @@ -2067,8 +2073,9 @@ pub struct DidResolutionConfig { impl DatabaseConfig { /// Reject settings that would make the node unusable (TC-1317): a zero /// read-audit wait would answer every authorized read `read_audit_busy`, - /// and an idle-in-transaction timeout within the legacy upload hold would - /// fail legitimate writes (TC-2954). + /// an idle-in-transaction timeout within the legacy upload hold would + /// fail legitimate writes (TC-2954), and a timeout above PostgreSQL's + /// largest (`INT_MAX` ms) would fail every connection after startup. pub fn validate(&self) -> Result<(), &'static str> { if self.read_audit_wait_ms == 0 { return Err("database.read_audit_wait_ms must be greater than zero"); @@ -2081,18 +2088,39 @@ impl DatabaseConfig { "database.idle_in_transaction_timeout_ms must be 0 (off) or exceed the 240 s legacy upload hold", ); } + for (value, error) in [ + ( + self.idle_in_transaction_timeout_ms, + "database.idle_in_transaction_timeout_ms must be at most 2147483647", + ), + ( + self.statement_timeout_ms, + "database.statement_timeout_ms must be at most 2147483647", + ), + ( + self.bulk_statement_timeout_ms, + "database.bulk_statement_timeout_ms must be at most 2147483647", + ), + ] { + if value > POSTGRES_MAX_TIMEOUT_MS { + return Err(error); + } + } Ok(()) } /// The statement that sets the pool's idle-in-transaction timeout on a new - /// connection, or `None` when it is off. + /// connection, or `None` when it is off or the statement timeout mode is + /// `local` (a transaction-pooling proxy). pub fn idle_in_transaction_statement(&self) -> Option { - (self.idle_in_transaction_timeout_ms > 0).then(|| { - format!( - "SET idle_in_transaction_session_timeout = {}", - self.idle_in_transaction_timeout_ms - ) - }) + (self.idle_in_transaction_timeout_ms > 0 + && self.statement_timeout_mode != StatementTimeoutMode::Local) + .then(|| { + format!( + "SET idle_in_transaction_session_timeout = {}", + self.idle_in_transaction_timeout_ms + ) + }) } /// The pool's PostgreSQL `statement_timeout` policy. @@ -2337,7 +2365,14 @@ mod tests { Config::default().database.idle_in_transaction_timeout_ms, 300_000 ); - for (timeout_ms, ok) in [(0, true), (1, false), (240_000, false), (240_001, true)] { + for (timeout_ms, ok) in [ + (0, true), + (1, false), + (240_000, false), + (240_001, true), + (2_147_483_647, true), + (2_147_483_648, false), + ] { let mut config = Config::default(); config.database.idle_in_transaction_timeout_ms = timeout_ms; assert_eq!( @@ -2348,6 +2383,39 @@ mod tests { } } + /// TC-2954: a statement timeout above PostgreSQL's largest is refused at + /// startup instead of failing every connection after the old node has + /// stopped. + #[test] + fn startup_rejects_a_statement_timeout_postgres_cannot_hold() { + for (base, bulk, ok) in [ + (2_147_483_647, 2_147_483_647, true), + (2_147_483_648, 120_000, false), + (30_000, 2_147_483_648, false), + ] { + let mut config = Config::default(); + config.database.statement_timeout_ms = base; + config.database.bulk_statement_timeout_ms = bulk; + assert_eq!( + crate::resolve_runtime_config(&config).is_ok(), + ok, + "{base} {bulk}" + ); + } + } + + /// TC-2954: behind a transaction-pooling proxy (`local` mode) the + /// session-level idle-in-transaction timeout is not set. + #[test] + fn the_idle_in_transaction_timeout_is_not_set_in_local_mode() { + let mut database = Config::default().database; + assert!(database.idle_in_transaction_statement().is_some()); + database.statement_timeout_mode = StatementTimeoutMode::Local; + assert_eq!(database.idle_in_transaction_statement(), None); + database.statement_timeout_mode = StatementTimeoutMode::Off; + assert!(database.idle_in_transaction_statement().is_some()); + } + /// TC-1491: off by default, 60 s / 8 MiB for every tier, and nothing is /// validated while the feature is off. #[test] diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index b483c0aee..3b43df0a5 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -517,6 +517,12 @@ pub async fn postgres_pool_with_recovery( let statement_timeouts = database.statement_timeouts(); let session_statement = statement_timeouts.session_statement(); let idle_in_transaction_statement = database.idle_in_transaction_statement(); + // The value in effect: 0 when off or in `local` statement-timeout mode. + let idle_in_transaction_timeout_ms = if idle_in_transaction_statement.is_some() { + database.idle_in_transaction_timeout_ms + } else { + 0 + }; let pool = recovery .install( connect_opts @@ -546,7 +552,7 @@ pub async fn postgres_pool_with_recovery( statement_timeout_mode = ?statement_timeouts.mode, statement_timeout_ms = statement_timeouts.base.as_millis() as u64, bulk_statement_timeout_ms = statement_timeouts.bulk.as_millis() as u64, - idle_in_transaction_timeout_ms = database.idle_in_transaction_timeout_ms, + idle_in_transaction_timeout_ms, retry_window_ms = database.retry_window_ms, tcp_keepalive = keepalive.is_some(), tcp_keepalive_idle_secs = keepalive.map(|k| k.idle.as_secs()),