feat(config): make startup bounds configurable

.github/workflows/fuzz.yml

@@ -51,9 +51,18 @@ jobs:
             dictionary: fuzz/dictionaries/sql.dict
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      # A pinned nightly keeps the build cache key stable across days. Move
+      # the pin forward on purpose, not by the calendar.
       - uses: dtolnay/rust-toolchain@4fd1da8b0805d2d2e936788875a7d65dbd677dc2
         with:
-          toolchain: nightly
+          toolchain: nightly-2026-10-01
+      # A cold ASan build takes most of the job's time limit. One cache per
+      # target, keyed on the toolchain and fuzz/Cargo.lock. The weekly run on
+      # main warms it for every pull request.
+      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        with:
+          workspaces: fuzz -> target
+          key: ${{ matrix.target.name }}
       # All six matrix jobs install the same pinned binary; build it once.
       - name: Cache cargo-fuzz
         id: cache-cargo-fuzz

docs/getting-started.md

@@ -629,7 +629,24 @@ Every value must be positive. `verdict_stall_warn_ms` has a floor of `4`.
 After a restart, a scheduler re-applies epochs up to the highest epoch it
 applied before. The client gateway opens only after every scheduler on the
 node reaches that epoch. The wait shares the data-group recovery bound, and
-startup fails when the bound expires.
+startup fails when no scheduler makes progress for that long.
+
+**Startup bounds** (config file only):
+
+| Config field                                    | Default |
+| ----------------------------------------------- | ------- |
+| `tuning.startup.raft_ready_timeout_ms`          | `30000` |
+| `tuning.startup.data_group_recovery_timeout_ms` | `60000` |
+
+Boot fails when a wait makes no progress for its bound. A long replay that
+keeps applying entries never fails.
+
+- `raft_ready_timeout_ms` bounds the metadata raft group's replay.
+- `data_group_recovery_timeout_ms` bounds the data raft groups' replay, then
+  the Calvin scheduler catch-up.
+
+Each value must be between `1` and `86400000` (one day). Both keys belong under
+`[tuning.startup]`. Startup refuses them under `[server]`.
 
 **Observability settings:**
 

nodedb-test-support/src/cluster_harness/node/lifecycle/spawn_full.rs

@@ -340,7 +340,10 @@ impl TestClusterNode {
         // the harness starts nodes one at a time, and a group elects only
         // once its peers run. It sets `data_groups_replayed`, which gates
         // the forget of a departed vShard's Calvin state.
-        nodedb::bootstrap::data_group_recovery::spawn_data_group_recovery(&shared);
+        nodedb::bootstrap::data_group_recovery::spawn_data_group_recovery(
+            &shared,
+            nodedb_types::config::tuning::StartupTuning::default().data_group_recovery_timeout(),
+        );
 
         // CRDT constraint reconcile loop (leader-gated). The production server
         // spawns this from `spawn_background_loops`, which the harness does not

nodedb-test-support/src/single_node.rs

@@ -118,8 +118,16 @@ pub async fn start(
     };
     // The harness cores replay their WAL before they report ready, so no
     // replay receiver is owed here.
-    nodedb::bootstrap::cluster_ready::await_cluster_ready(shared, ready_rx, Vec::new(), gates)
-        .await?;
+    let startup = nodedb_types::config::tuning::StartupTuning::default();
+    nodedb::bootstrap::cluster_ready::await_cluster_ready(
+        shared,
+        ready_rx,
+        Vec::new(),
+        gates,
+        startup.raft_ready_timeout(),
+        startup.data_group_recovery_timeout(),
+    )
+    .await?;
     Ok(raft)
 }
 

nodedb-types/src/config/tuning/config.rs

@@ -14,6 +14,7 @@ use super::network::{BridgeTuning, ClusterTransportTuning, NetworkTuning, WalTun
 use super::promql::PromqlTuning;
 use super::scheduler::SchedulerTuning;
 use super::shutdown::ShutdownTuning;
+use super::startup::StartupTuning;
 
 /// Top-level tuning configuration.
 ///
@@ -57,6 +58,8 @@ pub struct TuningConfig {
     pub calvin: CalvinTuning,
     #[serde(default)]
     pub promql: PromqlTuning,
+    #[serde(default)]
+    pub startup: StartupTuning,
 }
 
 impl TuningConfig {
@@ -194,4 +197,20 @@ doc_cache_entries = 8192
         assert_eq!(cfg.memory.overflow_max_bytes, 2 * 1024 * 1024 * 1024);
         assert_eq!(cfg.memory.doc_cache_entries, 8192);
     }
+
+    #[test]
+    fn startup_bounds_are_read_from_the_aggregate() {
+        let cfg: TuningConfig = toml::from_str("").expect("deserialize");
+        assert_eq!(cfg.startup.raft_ready_timeout_ms, 30_000);
+        assert_eq!(cfg.startup.data_group_recovery_timeout_ms, 60_000);
+
+        let toml_str = r#"
+[startup]
+raft_ready_timeout_ms = 1234
+data_group_recovery_timeout_ms = 5678
+"#;
+        let cfg: TuningConfig = toml::from_str(toml_str).expect("deserialize");
+        assert_eq!(cfg.startup.raft_ready_timeout_ms, 1234);
+        assert_eq!(cfg.startup.data_group_recovery_timeout_ms, 5678);
+    }
 }

nodedb-types/src/config/tuning/mod.rs

@@ -11,6 +11,7 @@ mod network;
 mod promql;
 mod scheduler;
 mod shutdown;
+mod startup;
 
 pub use bitemporal::BitemporalTuning;
 pub use calvin::CalvinTuning;
@@ -27,3 +28,7 @@ pub use network::{BridgeTuning, ClusterTransportTuning, NetworkTuning, WalTuning
 pub use promql::PromqlTuning;
 pub use scheduler::SchedulerTuning;
 pub use shutdown::ShutdownTuning;
+pub use startup::{
+    DEFAULT_DATA_GROUP_RECOVERY_TIMEOUT_MS, DEFAULT_RAFT_READY_TIMEOUT_MS,
+    DataGroupRecoveryTimeout, RaftReadyTimeout, StartupTuning,
+};

nodedb-types/src/config/tuning/startup.rs

@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: Apache-2.0
+
+//! Startup tuning: the stall bounds of the boot readiness waits.
+
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+
+/// Default stall bound of the metadata group's replay wait.
+pub const DEFAULT_RAFT_READY_TIMEOUT_MS: u64 = 30_000;
+
+/// Default stall bound of the data groups' replay wait and the Calvin
+/// scheduler catch-up wait.
+pub const DEFAULT_DATA_GROUP_RECOVERY_TIMEOUT_MS: u64 = 60_000;
+
+/// Stall bound of the metadata group's replay wait.
+///
+/// A distinct type from [`DataGroupRecoveryTimeout`], so the two bounds
+/// cannot swap places as adjacent arguments.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct RaftReadyTimeout(pub Duration);
+
+/// Stall bound of the data groups' replay wait and the Calvin scheduler
+/// catch-up wait.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct DataGroupRecoveryTimeout(pub Duration);
+
+impl From<Duration> for RaftReadyTimeout {
+    fn from(value: Duration) -> Self {
+        Self(value)
+    }
+}
+
+impl From<Duration> for DataGroupRecoveryTimeout {
+    fn from(value: Duration) -> Self {
+        Self(value)
+    }
+}
+
+fn default_raft_ready_timeout_ms() -> u64 {
+    DEFAULT_RAFT_READY_TIMEOUT_MS
+}
+
+fn default_data_group_recovery_timeout_ms() -> u64 {
+    DEFAULT_DATA_GROUP_RECOVERY_TIMEOUT_MS
+}
+
+/// `[tuning.startup]`: how long each boot wait can go without progress.
+///
+/// A wait that keeps progressing never fails, however long the replay takes.
+/// The server range-checks both values at load. An unknown key is refused, so
+/// a misspelt bound never leaves the default in force silently.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct StartupTuning {
+    /// Stall bound of the metadata group's replay. Every applied entry resets
+    /// it. Default: 30_000.
+    #[serde(default = "default_raft_ready_timeout_ms")]
+    pub raft_ready_timeout_ms: u64,
+
+    /// Stall bound of the data groups' replay, then of the Calvin scheduler
+    /// catch-up. An election, a commit or apply advance, a scheduler epoch, or
+    /// a finished group or scheduler resets it. Default: 60_000.
+    #[serde(default = "default_data_group_recovery_timeout_ms")]
+    pub data_group_recovery_timeout_ms: u64,
+}
+
+impl Default for StartupTuning {
+    fn default() -> Self {
+        Self {
+            raft_ready_timeout_ms: default_raft_ready_timeout_ms(),
+            data_group_recovery_timeout_ms: default_data_group_recovery_timeout_ms(),
+        }
+    }
+}
+
+impl StartupTuning {
+    /// Stall bound of the metadata group's replay wait.
+    pub fn raft_ready_timeout(&self) -> RaftReadyTimeout {
+        RaftReadyTimeout(Duration::from_millis(self.raft_ready_timeout_ms))
+    }
+
+    /// Stall bound of the data groups' replay and Calvin catch-up waits.
+    pub fn data_group_recovery_timeout(&self) -> DataGroupRecoveryTimeout {
+        DataGroupRecoveryTimeout(Duration::from_millis(self.data_group_recovery_timeout_ms))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn defaults_are_thirty_and_sixty_seconds() {
+        let cfg = StartupTuning::default();
+        assert_eq!(
+            cfg.raft_ready_timeout(),
+            RaftReadyTimeout(Duration::from_secs(30))
+        );
+        assert_eq!(
+            cfg.data_group_recovery_timeout(),
+            DataGroupRecoveryTimeout(Duration::from_secs(60))
+        );
+    }
+
+    #[test]
+    fn one_bound_set_leaves_the_other_at_its_default() {
+        let cfg: StartupTuning =
+            toml::from_str("raft_ready_timeout_ms = 1234").expect("deserialize");
+        assert_eq!(cfg.raft_ready_timeout_ms, 1234);
+        assert_eq!(
+            cfg.data_group_recovery_timeout_ms,
+            DEFAULT_DATA_GROUP_RECOVERY_TIMEOUT_MS
+        );
+    }
+
+    #[test]
+    fn an_unknown_key_is_refused() {
+        let error = toml::from_str::<StartupTuning>("data_group_recovery_timeout = 1800000")
+            .expect_err("a key without the _ms suffix is not a bound");
+        let message = error.to_string();
+        assert!(message.contains("data_group_recovery_timeout"), "{message}");
+        assert!(
+            message.contains("data_group_recovery_timeout_ms"),
+            "the message must list the keys that exist: {message}"
+        );
+    }
+}

nodedb-wal/src/error.rs

@@ -31,6 +31,22 @@ pub enum WalError {
     #[error("unsupported WAL format version {version} (supported: {supported})")]
     UnsupportedVersion { version: u16, supported: u16 },
 
+    /// A segment holds records in a WAL format version this build cannot read.
+    ///
+    /// The same finding as [`WalError::UnsupportedVersion`], reported by the
+    /// callers that know which file carries it. A reader sees one header at a
+    /// time and holds no path, and a version alone does not tell an operator
+    /// which segment to act on.
+    #[error(
+        "WAL segment '{path}' holds records in WAL format version {version}; this build reads \
+         version {supported}"
+    )]
+    SegmentFormatVersion {
+        path: String,
+        version: u16,
+        supported: u16,
+    },
+
     /// Unknown required record type encountered during replay.
     /// Optional unknown record types are safely skipped.
     #[error("unknown required record type {record_type} at LSN {lsn}")]
@@ -214,4 +230,24 @@ pub enum WalError {
     },
 }
 
+impl WalError {
+    /// Name the segment a version gap was found in.
+    ///
+    /// A reader reports [`WalError::UnsupportedVersion`] from header validation
+    /// alone and holds no path. The callers that opened the file do, and only
+    /// they can turn the finding into something an operator can act on. Every
+    /// other error passes through unchanged.
+    #[cfg(not(target_arch = "wasm32"))]
+    pub(crate) fn with_segment_path(self, path: &std::path::Path) -> Self {
+        match self {
+            Self::UnsupportedVersion { version, supported } => Self::SegmentFormatVersion {
+                path: path.display().to_string(),
+                version,
+                supported,
+            },
+            other => other,
+        }
+    }
+}
+
 pub type Result<T> = std::result::Result<T, WalError>;