From 6c672b4e1f083a1b79b66850abf4923bbea9babc Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 7 Nov 2026 12:00:35 -0500 Subject: [PATCH 1/3] fix(native): keep interrupted restarts recoverable under concurrent ps A `hack up` or `hack restart` could fail with `provider_busy` while `hack ps` or `hack logs` polled, because the reads hold the exclusive provider lease briefly and several mutating steps took it without waiting. When the restore of a retained run hit that refusal, its pre-admission cleanup also took the lease without waiting and exited with the foreground publication still bound. From then on `up` refused with `graph_retained_startup`, `down` reported the project stopped, `doctor --native-cleanup inspect` saw nothing pending and `down --recover` was refused. Mutating steps now wait out a brief holder within a bound, and only acquisition is retried: - retained-start eligibility (5 s), as runtime startup already did; - restore admission, within its restore budget and cancellation; - the foreground's pre-admission cleanup and owned cleanup (30 s); - inspections inside up, restart, down and recovery (15 s). ps and logs keep their 2 s budget and still report a busy runtime. A remaining foreground publication now names the next step instead of the generic refusal: a running owner (stop it), an owner that exited while its services ran (`hack down --recover`), or an owner that exited before its restore was admitted (reported; no supported retirement). `hack down --recover` now works for native image projects. It selects the run's exact receipt with the new read-only `graph foreground-owner --run-id RUN --json` and stops an exited ready owner's services through the existing same-boot `recover-live-owner`, retaining data. If that run's frontend also exited, it prints the exact `hack restart --recover-frontend --expect-finalization-attempt` step. Frontend recovery now accepts a project that never recorded lifecycle processes (no lifecycle state file), as the lifecycle runtime does. Refs HACK-1954 Co-Authored-By: Claude Opus 5.5 --- docs/guides/native-candidate.md | 18 ++ docs/reference/cli.md | 2 +- packages/runtime-core/src/graph_cli.rs | 17 +- .../src/provider/graph/foreground.rs | 5 +- .../provider/graph/foreground/transport.rs | 70 ++++++ .../runtime-core/src/provider/graph/mod.rs | 24 ++ .../src/provider/graph/restore.rs | 17 ++ .../src/provider/graph/retained_startup.rs | 105 +++++++- .../runtime-core/src/provider/lifecycle.rs | 9 +- packages/runtime-core/src/provider/state.rs | 58 +++++ src/backends/native-project-down.ts | 181 ++++++++++++- src/backends/native-project-inspect.ts | 21 +- src/backends/native-project-observe.ts | 2 + src/backends/native-project-recovery.ts | 7 +- src/commands/project.ts | 14 +- tests/native-project-down.test.ts | 237 ++++++++++++++++++ tests/native-project-inspect.test.ts | 56 ++++- tests/native-project-recovery.test.ts | 24 ++ 18 files changed, 845 insertions(+), 22 deletions(-) diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index ded85be95..2f98b172b 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -3014,6 +3014,24 @@ separately after restart. ### Explicit cleanup after a dead foreground owner +`hack up` and `hack restart` refuse a run whose foreground publication is still +present and name the next step: a running owner must be stopped with Ctrl-C or +`hack down`; an owner that exited while its services were running is stopped by +`hack down --recover`. That command selects the run's exact receipt with the +read-only `graph foreground-owner --run-id RUN --json` and then uses the +same-boot `graph recover-live-owner` described below, retaining data and +retiring the stale publication. If the frontend that owned the run also exited +before confirming its cleanup, `hack down --recover` prints the exact +`hack restart --recover-frontend --expect-finalization-attempt ATTEMPT` step. +An owner that exited before its restore was admitted is reported, with data +retained, but no command retires that attempt yet. + +Read-only `hack ps` and `hack logs` hold the provider lease only briefly. Native +startup, retained-start eligibility, restore admission, inspections inside +mutating commands and owned foreground cleanup wait out such a holder within a +bound instead of failing, so polling status during `up` or `restart` does not +make them fail or strand a foreground publication. + For a fully ready graph whose foreground owner died while its pool remains on the same boot, `graph recover-live-owner --run-id RUN --expect-receipt SHA256` selects only that graph. It requires matching dead foreground and relay-owner identities, diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 146b40069..3b73ac020 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -194,7 +194,7 @@ hack down [options] | `--remove-data` | Native backend: delete a stopped project's owned data volumes so the next up starts fresh (asks first) | | `--yes` | Confirm --prune-caches or --remove-data without prompting | | `--json` | Output JSON (machine-readable) | -| `--recover` | Explicitly reconcile an interrupted native Compose stop while retaining data | +| `--recover` | Explicitly reconcile an interrupted native stop, or stop the services of a native run whose foreground owner exited, while retaining data | | `--no-interactive` | Never prompt: apply documented defaults or fail with E_INTERACTIVE_REQUIRED (also via HACK_NO_INTERACTIVE=1) | | `--help, -h` | Show help | | `--version, -v` | Show version | diff --git a/packages/runtime-core/src/graph_cli.rs b/packages/runtime-core/src/graph_cli.rs index 61b5377db..90410c78c 100644 --- a/packages/runtime-core/src/graph_cli.rs +++ b/packages/runtime-core/src/graph_cli.rs @@ -20,7 +20,7 @@ use std::{collections::BTreeMap, path::Path, time::Duration}; fn invalid() -> CandidateError { CandidateError::new( "graph_arguments", - "Use graph run|restart|restore with --project, --file, --expect-plan, --run-id and --ready service=started|healthy|completed; planning actions accept optional --branch . Source mounts may select --source-revision. Fresh run/serve may explicitly select --release-initializer-cache service for quiescent guest page/dentry cache release (package contents retained); explicit --live-source binds directory mounts to that initial acknowledged workspace revision; inspect/reconcile/cleanup/archive/export/reconcile-export/prune require --run-id. Cleanup alone may use --remove-data. Dead-owner recover-cleanup and same-boot recover-live-owner require --expect-receipt; retire-recovered-publisher requires --expect-owner. A missing publisher lock requires inspect-missing-publication-lock --run-id and explicitly selected retire-missing-publication-lock --run-id --expect-selection --accept-quiesced-candidate-home. Foreground serve may opt into --auto-dependency-slots for pool-wide transport allocation; dependency-reservations inspects claims and recover-dependency-reservation --run-id --expect-reservation releases a verified dead pre-admission claim. Foreground serve additionally requires --dependencies and --expect-dependencies and accepts explicit --environment-stdin and --route-slot service=index|auto for reviewed local routes; dependency-plan requires --dependencies; dependency-discover requires --host-port and exactly one of --executable or --ancestor-pid; owner-status requires --run-id. Fresh foreground owner-restore requires exactly --run-id, --expect-plan, --expect-generation and --environment-stdin, plus optional --json. Bridge reservation requires --run-id, --service, --slot and --expect-generation; start/release require --run-id, --slot and --expect-reservation. bridges/reconcile-bridges require --run-id. Foreground publish-bridge requires --run-id, --slot, --expect-reservation and exactly one of --port or --unix (no --json); --unix accepts up to eight --hostname claims; unpublish-bridge requires --run-id and --expect-reservation.", + "Use graph run|restart|restore with --project, --file, --expect-plan, --run-id and --ready service=started|healthy|completed; planning actions accept optional --branch . Source mounts may select --source-revision. Fresh run/serve may explicitly select --release-initializer-cache service for quiescent guest page/dentry cache release (package contents retained); explicit --live-source binds directory mounts to that initial acknowledged workspace revision; inspect/reconcile/cleanup/archive/export/reconcile-export/prune require --run-id. Cleanup alone may use --remove-data. foreground-owner --run-id selects a run's receipt digest and publication state read-only. Dead-owner recover-cleanup and same-boot recover-live-owner require --expect-receipt; retire-recovered-publisher requires --expect-owner. A missing publisher lock requires inspect-missing-publication-lock --run-id and explicitly selected retire-missing-publication-lock --run-id --expect-selection --accept-quiesced-candidate-home. Foreground serve may opt into --auto-dependency-slots for pool-wide transport allocation; dependency-reservations inspects claims and recover-dependency-reservation --run-id --expect-reservation releases a verified dead pre-admission claim. Foreground serve additionally requires --dependencies and --expect-dependencies and accepts explicit --environment-stdin and --route-slot service=index|auto for reviewed local routes; dependency-plan requires --dependencies; dependency-discover requires --host-port and exactly one of --executable or --ancestor-pid; owner-status requires --run-id. Fresh foreground owner-restore requires exactly --run-id, --expect-plan, --expect-generation and --environment-stdin, plus optional --json. Bridge reservation requires --run-id, --service, --slot and --expect-generation; start/release require --run-id, --slot and --expect-reservation. bridges/reconcile-bridges require --run-id. Foreground publish-bridge requires --run-id, --slot, --expect-reservation and exactly one of --port or --unix (no --json); --unix accepts up to eight --hostname claims; unpublish-bridge requires --run-id and --expect-reservation.", ) } pub fn command(candidate: &Candidate, args: &[&str]) -> Result { @@ -123,6 +123,21 @@ pub fn command(candidate: &Candidate, args: &[&str]) -> Result run, + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::foreground_owner(candidate, run); + } + #[cfg(not(target_os = "macos"))] + { + let _ = run; + return Err(invalid()); + } + } if *action == "retire-acknowledged-publisher" { let (run, owner, receipt, publisher) = match *args { [ diff --git a/packages/runtime-core/src/provider/graph/foreground.rs b/packages/runtime-core/src/provider/graph/foreground.rs index 0797bdaa9..2e5c61373 100644 --- a/packages/runtime-core/src/provider/graph/foreground.rs +++ b/packages/runtime-core/src/provider/graph/foreground.rs @@ -285,7 +285,10 @@ fn serve_input<'a>( // containers. It travels only in this reply and is never persisted. let error = error.with_startup_logs(super::startup_logs::capture(candidate, &run_id)); if fresh_generation.is_some() && !runtime.admission_started() { - let engine = Engine::connect_cleanup(candidate)?; + // Wait out a concurrent read-only holder of the lease: refusing here + // would exit with this attempt's publication still bound, which + // blocks every later start of the retained run. + let engine = Engine::connect_cleanup_wait(candidate)?; let (retained, root) = super::load(candidate, &engine, &run_id)?; if retained.phase == "stopped-data-retained" && std::fs::symlink_metadata(root.join("state.pending")) diff --git a/packages/runtime-core/src/provider/graph/foreground/transport.rs b/packages/runtime-core/src/provider/graph/foreground/transport.rs index 47f7421d8..8338ecec7 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport.rs @@ -843,6 +843,53 @@ pub(in crate::provider::graph) struct LegacyPublisher { pub process: ProcessIdentity, } +/// What a remaining foreground publication shows about its recorded process. +/// Presentation only: it names the next step for a refused start and grants no +/// retirement, cleanup or adoption authority. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(in crate::provider::graph) enum PublishedOwner { + /// Neither the owner record nor its control socket exists. + Absent, + /// The recorded owner process is alive with its exact identity. + Running, + /// The recorded owner process is gone and nothing listens on its socket. + Exited, + /// The publication is partial, foreign or otherwise unclassified. + Uncertain, +} +impl PublishedOwner { + pub(in crate::provider::graph) fn word(self) -> &'static str { + match self { + Self::Absent => "absent", + Self::Running => "running", + Self::Exited => "exited", + Self::Uncertain => "uncertain", + } + } +} +pub(in crate::provider::graph) fn published_owner( + candidate: &Candidate, + run: &str, +) -> Result { + let root = root(candidate, run)?; + let present = |name: &str| match fs::symlink_metadata(root.join(name)) { + Ok(_) => Ok(true), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(_) => Err(refused()), + }; + if !present("owner.json")? && !present("control.sock")? { + return Ok(PublishedOwner::Absent); + } + let Ok(pin) = Pin::read(candidate, run) else { + return Ok(PublishedOwner::Uncertain); + }; + Ok(match identity::alive(pin.record.process.pid) { + Ok(true) if pin.verify().is_ok() => PublishedOwner::Running, + Ok(false) if no_listener(&pin.root.join("control.sock")).is_ok() => PublishedOwner::Exited, + _ => PublishedOwner::Uncertain, + }) +} + /// Select an exact dead current-device publisher without requiring its missing /// operation lock. This observation grants no file or guest effect. pub(in crate::provider::graph) fn dead_publisher_without_lock( @@ -879,6 +926,29 @@ pub(in crate::provider::graph) fn fixture_dead_publisher_without_lock( .unwrap(); fs::remove_file(root.join("operation.lock")).unwrap(); } +/// Bind a retired run's next publication in this test process. The returned value +/// keeps it alive (its owner is this process) until dropped. +#[cfg(test)] +pub(in crate::provider::graph) fn fixture_live_publication( + candidate: &Candidate, + run: &str, +) -> Box { + Box::new(Publication::bind_retired(candidate, run).unwrap()) +} +/// Leave a retired run's next publication behind with an exited owner process: +/// the listener is closed and the record names a process that does not exist. +#[cfg(test)] +pub(in crate::provider::graph) fn fixture_exited_publication(candidate: &Candidate, run: &str) { + drop(Publication::bind_retired(candidate, run).unwrap()); + let root = root(candidate, run).unwrap(); + let mut record: Record = state::read(&root.join("owner.json")).unwrap(); + record.process.pid = 2_000_000; + fs::write( + root.join("owner.json"), + serde_json::to_vec(&record).unwrap(), + ) + .unwrap(); +} #[cfg(test)] pub(in crate::provider::graph) fn fixture_next_publication(candidate: &Candidate, run: &str) { let mut publication = Publication::bind(candidate, run).unwrap(); diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 1fb72729c..74f625ce8 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -1668,6 +1668,30 @@ fn endpoint_network<'a>( Ok(Some(network)) } +/// Read-only selection for an explicit dead-owner recovery such as `hack down --recover`: +/// the run's receipt phase, the SHA-256 of its exact receipt bytes and what its +/// foreground publication shows (`absent`, `running`, `exited` or `uncertain`). +/// +/// It takes no lease, contacts no guest and grants no authority. The digest only lets +/// a caller select the receipt it observed; each recovery command revalidates its own +/// evidence (owner death, endpoints, inventories) against that selection. +#[cfg(target_os = "macos")] +pub fn foreground_owner(candidate: &Candidate, run: &str) -> Result { + use sha2::{Digest, Sha256}; + let owner = state::Owner::load(candidate)?; + let (receipt, root) = load_at(directory(candidate, run)?, run, &owner.token)?; + let bytes = host_pin_recovery::read_raw(&root.join("state.json"), 1024 * 1024)?; + let publication = foreground::transport::published_owner(candidate, run)?; + Ok(json!({ + "version": 1, + "run": receipt.run, + "owner": receipt.owner, + "phase": receipt.phase, + "receipt_sha256": format!("{:x}", Sha256::digest(&bytes)), + "foreground": publication.word(), + })) +} + /// Read-only recovery inspection; historical ready phase is not a current health assertion. /// Read-only inspection of one owned graph attempt. /// diff --git a/packages/runtime-core/src/provider/graph/restore.rs b/packages/runtime-core/src/provider/graph/restore.rs index 5168e6736..c13e7d98c 100644 --- a/packages/runtime-core/src/provider/graph/restore.rs +++ b/packages/runtime-core/src/provider/graph/restore.rs @@ -251,6 +251,23 @@ fn restore_inputs( if options.timeout.is_zero() || options.timeout > Duration::from_secs(600) { return Err(error("graph_budget", "Invalid restore timeout.")); } + // Nothing above has effects, and acquisition admits nothing: wait out a concurrent + // read-only holder of the lease (the inspection behind `hack ps`) within the restore + // budget instead of failing the restore, as graph startup does. Cancellation stops + // the wait before admission. + #[cfg(target_os = "macos")] + let engine = { + let budget = std::time::Instant::now() + options.timeout; + let deadline = fresh + .as_ref() + .map_or(budget, |fresh| fresh.deadline.min(budget)); + Engine::connect_until(candidate, deadline, || { + startup + .as_ref() + .is_some_and(|driver| driver.check_cancelled().is_err()) + })? + }; + #[cfg(not(target_os = "macos"))] let engine = Engine::connect(candidate)?; if engine.guest().profile() != super::super::Profile::Development { return Err(error( diff --git a/packages/runtime-core/src/provider/graph/retained_startup.rs b/packages/runtime-core/src/provider/graph/retained_startup.rs index 4472ab18d..4c7a46729 100644 --- a/packages/runtime-core/src/provider/graph/retained_startup.rs +++ b/packages/runtime-core/src/provider/graph/retained_startup.rs @@ -12,6 +12,33 @@ fn refused() -> CandidateError { ) } +/// A remaining foreground publication blocks retained startup. Name the next +/// step when its recorded owner is classified; the classification grants nothing. +fn published_owner_refused(candidate: &Candidate, run: &str) -> CandidateError { + use foreground::transport::{PublishedOwner, published_owner}; + match published_owner(candidate, run) { + Ok(PublishedOwner::Running) => error( + "graph_foreground_owner_running", + "Another foreground hack up or hack restart still owns this run. Stop it with Ctrl-C in its terminal, or run hack down, then start the project again.", + ), + Ok(PublishedOwner::Exited) if exited_ready(candidate, run) => error( + "graph_foreground_owner_exited", + "The previous foreground owner of this run exited while its services were running and did not retire its publication. Run hack down --recover to stop the orphaned services with data retained, then hack up.", + ), + Ok(PublishedOwner::Exited) => error( + "graph_foreground_owner_exited_unadmitted", + "The previous foreground owner of this run exited before its start was confirmed and did not retire its publication. Data is retained, but no supported command can prove and retire that attempt yet; keep the native home unchanged for inspection.", + ), + _ => refused(), + } +} +fn exited_ready(candidate: &Candidate, run: &str) -> bool { + Owner::load(candidate) + .ok() + .and_then(|owner| load_at(directory(candidate, run).ok()?, run, &owner.token).ok()) + .is_some_and(|(receipt, _)| receipt.phase == "ready-observed") +} + fn no_file(root: &Path, name: &str) -> Result<(), CandidateError> { match fs::symlink_metadata(root.join(name)) { Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), @@ -88,8 +115,8 @@ impl Guard { run: &str, expected: Option<&str>, ) -> Result { - let retired = - foreground::transport::Retired::acquire(candidate, run)?.ok_or_else(refused)?; + let retired = foreground::transport::Retired::acquire(candidate, run)? + .ok_or_else(|| published_owner_refused(candidate, run))?; let (owner, receipt) = checked(candidate, run)?; let selected = selection(&owner, &receipt)?; if expected.is_some_and(|value| !hex(value, 64) || value != selected) { @@ -117,7 +144,12 @@ impl Guard { /// values or restore authority are returned. An absent/uncertain owner refuses. pub fn preflight(candidate: &Candidate, run: &str) -> Result { let guard = Guard::acquire(candidate, run, None)?; - let _lease = state::Lock::acquire_existing(&candidate.state_root.join("run/smolvm"))?; + // Eligibility is read-only: wait out a brief holder such as the inspection behind + // `hack ps`, as runtime startup does, rather than failing the `up` that asked. + let _lease = state::Lock::acquire_existing_waiting( + &candidate.state_root.join("run/smolvm"), + lifecycle::STARTUP_LEASE_WAIT, + )?; guard.verify(candidate)?; let (owner, receipt) = checked(candidate, run)?; Ok( @@ -421,6 +453,73 @@ mod tests { ); } } + #[test] + fn a_remaining_publication_names_its_owner_state_without_effects() { + let state = |fixture: &Fixture| { + let owner = fixture.candidate.state_root.join("run/smolvm/owner.json"); + ( + fs::read(owner).unwrap(), + fs::read(fixture.graph().join("state.json")).unwrap(), + ) + }; + let refusal = |fixture: &Fixture| { + let error = preflight(&fixture.candidate, &fixture.run).unwrap_err(); + assert!( + !fixture + .candidate + .state_root + .join("run/smolvm/admission.json") + .exists() + ); + error.code + }; + // A live foreground owner (this process) still holds the run. + let fixture = Fixture::new(); + let live = + foreground::transport::fixture_live_publication(&fixture.candidate, &fixture.run); + let before = state(&fixture); + assert_eq!(refusal(&fixture), "graph_foreground_owner_running"); + assert_eq!(state(&fixture), before); + drop(live); + // An owner that exited before its restore was admitted: the receipt is the + // unchanged stopped one, and no supported recovery is claimed. + let fixture = Fixture::new(); + foreground::transport::fixture_exited_publication(&fixture.candidate, &fixture.run); + let before = state(&fixture); + assert_eq!( + refusal(&fixture), + "graph_foreground_owner_exited_unadmitted" + ); + assert_eq!(state(&fixture), before); + // An owner that exited while its graph was ready names `hack down --recover`. + let fixture = Fixture::new(); + let mut receipt = fixture.receipt(); + receipt.phase = "ready-observed".into(); + state::write(&fixture.graph().join("state.json"), &receipt).unwrap(); + foreground::transport::fixture_exited_publication(&fixture.candidate, &fixture.run); + let before = state(&fixture); + let error = preflight(&fixture.candidate, &fixture.run).unwrap_err(); + assert_eq!(error.code, "graph_foreground_owner_exited"); + assert!(error.message.contains("hack down --recover")); + assert_eq!(state(&fixture), before); + } + + #[test] + fn preflight_waits_out_a_brief_provider_lease_holder() { + let fixture = Fixture::new(); + let held = state::Lock::acquire_existing(&fixture.candidate.state_root.join("run/smolvm")) + .unwrap(); + let worker = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(200)); + drop(held); + }); + let started = std::time::Instant::now(); + let value = preflight(&fixture.candidate, &fixture.run).unwrap(); + worker.join().unwrap(); + assert!(started.elapsed() >= std::time::Duration::from_millis(150)); + assert_eq!(value["runtime_phase"], "stopped"); + } + #[test] fn preflight_selection_and_held_guard_refuse_a_valid_replacement_owner_before_up() { let mut fixture = Fixture::new(); diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index ff3ddcb5b..7509c2eb5 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -282,6 +282,13 @@ fn operation_lease( Ok((lock, current)) } +/// How long owned cleanup and recovery wait for a provider lease that another operation +/// holds, such as the `graph inspect` behind a polling `hack ps`. Acquisition admits nothing, +/// and a foreground owner that gives up its cleanup exits with its publication and services +/// still bound, so this waits longer than startup does. +#[cfg(target_os = "macos")] +pub(super) const OWNED_CLEANUP_LEASE_WAIT: Duration = Duration::from_secs(30); + /// How long ordinary startup waits for a provider lease that another operation holds, such as /// the `graph inspect` behind `hack ps`. pub(super) const STARTUP_LEASE_WAIT: Duration = Duration::from_secs(5); @@ -481,7 +488,7 @@ impl<'a> OwnedGuest<'a> { Self::connect_mode( candidate, false, - Some(Instant::now() + Duration::from_secs(5)), + Some(Instant::now() + OWNED_CLEANUP_LEASE_WAIT), ) } diff --git a/packages/runtime-core/src/provider/state.rs b/packages/runtime-core/src/provider/state.rs index 3b6bb835a..1c795fdca 100644 --- a/packages/runtime-core/src/provider/state.rs +++ b/packages/runtime-core/src/provider/state.rs @@ -110,6 +110,31 @@ impl Lock { Self::from_file(file) } + /// [`Self::acquire_existing`], waiting up to `wait` for another holder, such as the + /// short read-only inspection behind `hack ps`, to release the lock. Only acquisition + /// is retried: nothing is created or admitted while waiting, and on expiry the + /// ordinary `provider_busy` refusal is returned. + pub fn acquire_existing_waiting( + root: &Path, + wait: std::time::Duration, + ) -> Result { + let deadline = std::time::Instant::now() + wait; + loop { + match Self::acquire_existing(root) { + Err(error) + if error.code == "provider_busy" && std::time::Instant::now() < deadline => + { + std::thread::sleep( + deadline + .saturating_duration_since(std::time::Instant::now()) + .min(std::time::Duration::from_millis(50)), + ); + } + result => return result, + } + } + } + /// Adopt an exclusively created private lock file before publishing its /// pathname. The caller must fence the final pathname against this FD. pub(crate) fn from_file(file: File) -> Result { @@ -797,6 +822,39 @@ mod tests { fs::remove_dir_all(root).unwrap(); } #[test] + fn waiting_existing_acquisition_outlasts_a_brief_holder_and_refuses_a_lasting_one() { + let root = fixture(); + drop(Lock::acquire(&root).unwrap()); + // A brief holder, such as the inspection behind `hack ps`, is waited out. + let held = Lock::acquire_existing(&root).unwrap(); + let worker = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(150)); + drop(held); + }); + let started = std::time::Instant::now(); + let lock = + Lock::acquire_existing_waiting(&root, std::time::Duration::from_secs(2)).unwrap(); + worker.join().unwrap(); + assert!(started.elapsed() >= std::time::Duration::from_millis(100)); + // Still exclusive while held. + assert!(matches!(Lock::acquire_existing(&root), Err(e) if e.code == "provider_busy")); + drop(lock); + // A holder that outlasts the wait is refused as before. + let _held = Lock::acquire_existing(&root).unwrap(); + assert!(matches!( + Lock::acquire_existing_waiting(&root, std::time::Duration::from_millis(100)), + Err(e) if e.code == "provider_busy" + )); + drop(_held); + // Waiting never creates an absent lock. + fs::remove_file(root.join("operation.lock")).unwrap(); + assert!( + Lock::acquire_existing_waiting(&root, std::time::Duration::from_millis(100)).is_err() + ); + assert!(!root.join("operation.lock").exists()); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn symlink_and_hardlink_receipts_are_not_read() { let root = fixture(); let foreign = root.join("foreign.json"); diff --git a/src/backends/native-project-down.ts b/src/backends/native-project-down.ts index bd15a7440..048ab9c7c 100644 --- a/src/backends/native-project-down.ts +++ b/src/backends/native-project-down.ts @@ -87,6 +87,147 @@ function verify(value: unknown, run: NativeProjectRun, stopped: boolean): void { } } } +const RECEIPT_SHA = /^[a-f0-9]{64}$/; + +/** Only ESRCH proves a recorded frontend exited; a live or inaccessible PID does not. */ +function processExited(pid: number): boolean { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return isRecord(error) && error.code === "ESRCH"; + } +} +const FOREGROUND_STATES = new Set(["absent", "running", "exited", "uncertain"]); + +/** What `graph foreground-owner` selected; it grants no recovery authority. */ +type ForegroundOwnerSelection = { + readonly phase: string; + readonly receiptSha256: string; + readonly foreground: "absent" | "running" | "exited" | "uncertain"; +}; + +function ownerRecoveryRefused(): Error { + return new ClosedCliError( + "Native down --recover cannot confirm this run's foreground owner state; retained data and runtime state were not changed." + ); +} + +async function selectForegroundOwner(opts: { + readonly runtime: NativeRuntimeSelection; + readonly projectRoot: string; + readonly run: NativeProjectRun; + readonly invoke: typeof invokeNativeRuntime; +}): Promise { + const value = await opts.invoke({ + runtime: opts.runtime, + cwd: opts.projectRoot, + args: ["graph", "foreground-owner", "--run-id", opts.run.run, "--json"], + timeoutMs: 15_000, + }); + if ( + !isRecord(value) || + value.version !== 1 || + value.run !== opts.run.run || + value.owner !== opts.run.owner || + typeof value.phase !== "string" || + typeof value.receipt_sha256 !== "string" || + !RECEIPT_SHA.test(value.receipt_sha256) || + typeof value.foreground !== "string" || + !FOREGROUND_STATES.has(value.foreground) + ) { + throw ownerRecoveryRefused(); + } + return { + phase: value.phase, + receiptSha256: value.receipt_sha256, + foreground: value.foreground as ForegroundOwnerSelection["foreground"], + }; +} + +/** + * The owned stop for `hack down --recover`. A run whose foreground owner exited + * while its graph was ready is stopped by the runtime's same-boot dead-owner + * recovery, selected by the exact receipt observed here; it revalidates the + * owner's death, its endpoints and the inventories itself, retains data and + * retires the stale publication. A run without a remaining publication takes + * the ordinary cleanup. Every other state refuses with nothing changed. + */ +async function requestOwnedStop(opts: { + readonly runtime: NativeRuntimeSelection; + readonly projectRoot: string; + readonly run: NativeProjectRun; + readonly invoke: typeof invokeNativeRuntime; + readonly selection: ForegroundOwnerSelection | undefined; +}): Promise { + const selection = opts.selection; + if (selection === undefined || selection.foreground === "absent") { + try { + await opts.invoke({ + runtime: opts.runtime, + cwd: opts.projectRoot, + args: ["graph", "cleanup", "--run-id", opts.run.run, "--json"], + timeoutMs: 590_000, + }); + } catch (error) { + if ( + selection === undefined && + error instanceof NativeRuntimeRequestError && + error.nativeCode === "graph_owner_recovery" + ) { + throw new ClosedCliError( + "Native down cannot reach this run's foreground owner. If the hack up or hack restart that owned it has exited, run hack down --recover to stop its services with data retained. Nothing was stopped." + ); + } + throw error; + } + return; + } + const result = await opts.invoke({ + runtime: opts.runtime, + cwd: opts.projectRoot, + args: [ + "graph", + "recover-live-owner", + "--run-id", + opts.run.run, + "--expect-receipt", + selection.receiptSha256, + "--json", + ], + timeoutMs: 590_000, + }); + if ( + !isRecord(result) || + result.run !== opts.run.run || + result.recovered !== true || + result.data_retained !== true || + result.publisher_retired !== true + ) { + throw ownerRecoveryRefused(); + } +} + +/** Refuse `--recover` states that have no supported owned stop, before any hook. */ +function requireRecoverableOwner(selection: ForegroundOwnerSelection): void { + if (selection.foreground === "running") { + throw new ClosedCliError( + "Native down --recover found a running foreground owner for this run. Stop it with Ctrl-C in its terminal, or run hack down without --recover. Nothing was changed." + ); + } + if (selection.foreground === "uncertain") { + throw ownerRecoveryRefused(); + } + if ( + selection.foreground === "exited" && + selection.phase !== "ready-observed" + ) { + throw new ClosedCliError( + "Native down --recover found a foreground owner that exited before its start was confirmed. Data is retained, but no supported command can prove and retire that attempt yet; keep the native home unchanged for inspection. Nothing was changed." + ); + } +} + /** Retaining cleanup only. Success follows confirmed compute absence and the * original frontend's finalizers, including a previously recovered stop. */ @@ -120,6 +261,11 @@ export async function nativeProjectDown(opts: { }) => Promise; }; readonly invoke?: typeof invokeNativeRuntime; + /** + * `hack down --recover`: also stop a ready run whose foreground owner exited + * without retiring its publication (see {@link requestOwnedStop}). + */ + readonly recover?: boolean; /** Only the restart coordinator may defer to its already captured exact wait/recovery. */ readonly deferFinalization?: boolean; readonly finalizationTimeoutMs?: number; @@ -142,9 +288,27 @@ export async function nativeProjectDown(opts: { }); const initial = await inspect(); verify(initial, run, false); + const selection = opts.recover + ? await selectForegroundOwner({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + run, + invoke, + }) + : undefined; + if (selection) { + requireRecoverableOwner(selection); + } const finalization = opts.deferFinalization ? undefined : await captureFinalization({ scope: opts.scope, run }); + // `--recover`: a frontend that has exited cannot finish later, so its + // finalization is read once instead of awaited. + const frontendGone = + opts.recover === true && + finalization !== undefined && + finalization.version !== 1 && + processExited(finalization.pid); const wait = async () => { if (finalization === undefined) { return; @@ -154,10 +318,14 @@ export async function nativeProjectDown(opts: { scope: opts.scope, run, token: finalization, - timeoutMs: opts.finalizationTimeoutMs, + timeoutMs: frontendGone ? 1 : opts.finalizationTimeoutMs, }); } catch { - throw finalizationRefused(); + throw frontendGone + ? new ClosedCliError( + `Native down --recover: this run's services are stopped with data retained, but the hack up or hack restart that owned it exited before confirming its own cleanup. Run hack restart --recover-frontend --expect-finalization-attempt ${finalization.attempt} to finish that recovery and start the project.` + ) + : finalizationRefused(); } }; const hooks = opts.hooks; @@ -242,11 +410,12 @@ export async function nativeProjectDown(opts: { throw finalizationRefused(); } await opts.beforeCleanup?.(run); - await invoke({ + await requestOwnedStop({ runtime: opts.runtime, - cwd: opts.scope.projectRoot, - args: ["graph", "cleanup", "--run-id", run.run, "--json"], - timeoutMs: 590_000, + projectRoot: opts.scope.projectRoot, + run, + invoke, + selection, }); verify(await inspect(), run, true); await opts.retireHostProcesses?.(run); diff --git a/src/backends/native-project-inspect.ts b/src/backends/native-project-inspect.ts index 5f1419156..7964de6b2 100644 --- a/src/backends/native-project-inspect.ts +++ b/src/backends/native-project-inspect.ts @@ -19,6 +19,16 @@ export function isNativeProviderBusy(error: unknown): boolean { (error instanceof Error && error.message === BUSY) ); } +/** + * How long an inspection waits out lease contention by default. Inspections + * inside `up`, `restart`, `down` and recovery outlast a polling `hack ps` or + * `hack logs` (each holds the lease briefly), so a concurrent read never fails + * the mutating command. Those observers pass {@link OBSERVER_BUSY_WAIT_MS}. + */ +export const INSPECT_BUSY_WAIT_MS = 15_000; +/** Read-only observers report a busy runtime quickly instead of waiting. */ +export const OBSERVER_BUSY_WAIT_MS = 2000; + /** Concurrent cleanup observers share a provider lease. Retry only read-only lease contention. */ export async function inspectNativeProjectGraph(opts: { readonly runtime: NativeRuntimeSelection; @@ -26,8 +36,11 @@ export async function inspectNativeProjectGraph(opts: { readonly run: string; readonly invoke?: typeof invokeNativeRuntime; readonly signal?: AbortSignal; + /** Bound for retrying `provider_busy`; see {@link INSPECT_BUSY_WAIT_MS}. */ + readonly busyWaitMs?: number; }): Promise { - const deadline = performance.now() + 2000; + const deadline = + performance.now() + (opts.busyWaitMs ?? INSPECT_BUSY_WAIT_MS); for (;;) { try { return await (opts.invoke ?? invokeNativeRuntime)({ @@ -38,7 +51,11 @@ export async function inspectNativeProjectGraph(opts: { signal: opts.signal, }); } catch (error) { - if (!isNativeProviderBusy(error) || performance.now() >= deadline) { + if ( + !isNativeProviderBusy(error) || + performance.now() >= deadline || + opts.signal?.aborted + ) { throw error; } await Bun.sleep(50); diff --git a/src/backends/native-project-observe.ts b/src/backends/native-project-observe.ts index c4e685249..547835531 100644 --- a/src/backends/native-project-observe.ts +++ b/src/backends/native-project-observe.ts @@ -3,6 +3,7 @@ import { isRecord } from "../lib/guards.ts"; import { inspectNativeProjectGraph, isNativeProviderBusy, + OBSERVER_BUSY_WAIT_MS, } from "./native-project-inspect.ts"; import { loadNativeProjectRun, @@ -143,6 +144,7 @@ async function inspect(opts: Options, run: NativeProjectRun) { projectRoot: opts.scope.projectRoot, run: run.run, invoke: opts.invoke, + busyWaitMs: OBSERVER_BUSY_WAIT_MS, }), run ); diff --git a/src/backends/native-project-recovery.ts b/src/backends/native-project-recovery.ts index 638203c1d..22cfd6b34 100644 --- a/src/backends/native-project-recovery.ts +++ b/src/backends/native-project-recovery.ts @@ -82,6 +82,8 @@ async function retireOrphanedHttpsLock(path: string): Promise { } } +/** Lifecycle state records no process: absent (a project that never started one, + * as the lifecycle runtime reads it) or an exact empty entry list. */ async function noLifecycleEntries(projectDir: string): Promise { const path = join(projectDir, ".internal/lifecycle/state.json"); let file: Awaited>; @@ -90,7 +92,10 @@ async function noLifecycleEntries(projectDir: string): Promise { path, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW ); - } catch { + } catch (error) { + if (isRecord(error) && error.code === "ENOENT") { + return; + } throw refused(); } try { diff --git a/src/commands/project.ts b/src/commands/project.ts index 582c0ecc0..2e305a7b7 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -546,7 +546,7 @@ const downOptions = [ type: "boolean", long: "--recover", description: - "Explicitly reconcile an interrupted native Compose stop while retaining data", + "Explicitly reconcile an interrupted native stop, or stop the services of a native run whose foreground owner exited, while retaining data", } as const), ] as const; const restartOptions = [ @@ -6861,15 +6861,18 @@ function writeDownNotice(opts: { function refuseLegacyNativeRecovery(recover: boolean): void { if (recover) { - throw new CliUsageError( - "--recover requires a native-authored Compose instance." - ); + throw new CliUsageError("--recover requires the native backend."); } } /** Native down accepts --yes only to confirm --remove-data. */ function refuseNativeDownOptions(options: DownArgs["options"]): void { const removeData = options.removeData === true; + if (removeData && options.recoverNativeCompose) { + throw new CliUsageError( + "Native down --recover retains data; it cannot be combined with --remove-data." + ); + } if ( [ options.env, @@ -6921,7 +6924,6 @@ async function handleDown({ if (authored !== null) { return authored; } - refuseLegacyNativeRecovery(args.options.recoverNativeCompose); const native = resolveNativeRuntimeSelection(); if (native) { if (args.options.json) { @@ -6990,6 +6992,7 @@ async function handleDown({ const result = await nativeProjectDown({ runtime: native, scope, + recover: args.options.recoverNativeCompose, deferFinalization, retireHostProcesses: async () => { await stopLifecycleProcesses({ @@ -7033,6 +7036,7 @@ async function handleDown({ } return 0; } + refuseLegacyNativeRecovery(args.options.recoverNativeCompose); const json = args.options.json === true; if (!json) { return await runDownCommand({ ctx, args, json: false }); diff --git a/tests/native-project-down.test.ts b/tests/native-project-down.test.ts index ec3fb0ee1..4f17eed1c 100644 --- a/tests/native-project-down.test.ts +++ b/tests/native-project-down.test.ts @@ -25,6 +25,7 @@ import { removeNativeProjectRun, saveNativeProjectRun, } from "../src/backends/native-project-run.ts"; +import { NativeRuntimeRequestError } from "../src/backends/native-runtime-client.ts"; import { markSharedHelperInstanceStopping, type ProjectSharedHelperConfig, @@ -1050,3 +1051,239 @@ test("a committed native down marks shared helpers as stopping immediately befor expect(await helpers.generation()).not.toBeNull(); expect(await helpers.operation()).toStartWith("stopping:"); }); + +const receiptSha = "f".repeat(64); +function ownerSelection(foreground: string, phase = "ready-observed") { + return { + version: 1, + run: run.run, + owner: run.owner, + phase, + receipt_sha256: receiptSha, + foreground, + }; +} + +test("down without --recover names the recovery when the foreground owner is gone", async () => { + const opts = await fixture(); + const calls: string[] = []; + await expectClosedRefusal( + nativeProjectDown({ + ...opts, + invoke: async (request) => { + calls.push(request.args[1]!); + if (request.args[1] === "cleanup") { + throw new NativeRuntimeRequestError({ + message: "Native runtime request failed (graph_owner_recovery).", + nativeCode: "graph_owner_recovery", + }); + } + return snapshot(); + }, + }), + "run hack down --recover" + ); + expect(calls).toEqual(["inspect", "inspect", "cleanup"]); + expect(await loadNativeProjectRun(opts.scope)).toEqual(run); +}); + +test("down --recover stops a ready run whose foreground owner exited, selected by its exact receipt", async () => { + const opts = await fixture(); + const events: string[] = []; + let recovered = false; + const result = await nativeProjectDown({ + ...opts, + recover: true, + before: async () => { + events.push("before"); + }, + retireHostProcesses: async () => { + events.push("retire-host"); + }, + after: async () => { + events.push("after"); + }, + invoke: async (request) => { + events.push(request.args[1]!); + if (request.args[1] === "foreground-owner") { + expect(request.args).toEqual([ + "graph", + "foreground-owner", + "--run-id", + run.run, + "--json", + ]); + return ownerSelection("exited"); + } + if (request.args[1] === "recover-live-owner") { + expect(request.args).toEqual([ + "graph", + "recover-live-owner", + "--run-id", + run.run, + "--expect-receipt", + receiptSha, + "--json", + ]); + recovered = true; + return { + run: run.run, + phase: "stopped-data-retained", + recovered: true, + data_retained: true, + publisher_retired: true, + same_boot: true, + }; + } + expect(request.args[1]).toBe("inspect"); + return snapshot(recovered); + }, + }); + expect(result.status).toBe("stopped"); + expect(result.dataPreserved).toBe(true); + expect(events).toEqual([ + "inspect", + "foreground-owner", + "before", + "inspect", + "recover-live-owner", + "inspect", + "retire-host", + "after", + ]); + expect(await loadNativeProjectRun(opts.scope)).toEqual(run); +}); + +test("down --recover without a remaining publication takes the ordinary cleanup", async () => { + const opts = await fixture(); + const calls: string[] = []; + let cleaned = false; + const result = await nativeProjectDown({ + ...opts, + recover: true, + invoke: async (request) => { + calls.push(request.args[1]!); + if (request.args[1] === "foreground-owner") { + return ownerSelection("absent"); + } + if (request.args[1] === "cleanup") { + cleaned = true; + return {}; + } + return snapshot(cleaned); + }, + }); + expect(result.status).toBe("stopped"); + expect(calls).toEqual([ + "inspect", + "foreground-owner", + "inspect", + "cleanup", + "inspect", + ]); +}); + +test.each([ + { + name: "a running owner", + selection: ownerSelection("running"), + text: "found a running foreground owner", + }, + { + name: "an owner that exited before its start was confirmed", + selection: ownerSelection("exited", "stopped-data-retained"), + text: "no supported command can prove and retire that attempt yet", + }, + { + name: "an unclassified publication", + selection: ownerSelection("uncertain"), + text: "cannot confirm this run's foreground owner state", + }, + { + name: "a selection for another owner", + selection: { ...ownerSelection("exited"), owner: "9".repeat(32) }, + text: "cannot confirm this run's foreground owner state", + }, + { + name: "a malformed receipt digest", + selection: { ...ownerSelection("exited"), receipt_sha256: "short" }, + text: "cannot confirm this run's foreground owner state", + }, +])("down --recover refuses $name before hooks or any stop", async ({ + selection, + text, +}) => { + const opts = await fixture(); + const calls: string[] = []; + await expectClosedRefusal( + nativeProjectDown({ + ...opts, + recover: true, + before: async () => { + calls.push("before"); + }, + invoke: async (request) => { + calls.push(request.args[1]!); + if (request.args[1] === "foreground-owner") { + return selection; + } + expect(request.args[1]).toBe("inspect"); + return snapshot(selection.phase === "stopped-data-retained"); + }, + }), + text + ); + expect(calls).toEqual(["inspect", "foreground-owner"]); + expect(await loadNativeProjectRun(opts.scope)).toEqual(run); +}); + +test("down --recover names the exact frontend recovery when the owning frontend also exited", async () => { + const opts = await fixture(); + // A frontend that began its finalization and exited without completing it. + const module = join( + import.meta.dir, + "../src/backends/native-project-finalization.ts" + ); + const child = Bun.spawn( + [ + process.execPath, + "-e", + `const { beginNativeProjectFinalization } = await import(${JSON.stringify(module)}); + const lifetime = await beginNativeProjectFinalization(JSON.parse(process.argv[1])); + process.stdout.write(lifetime.token.attempt);`, + JSON.stringify({ scope: opts.scope, run }), + ], + { stdout: "pipe", stderr: "pipe" } + ); + const attempt = await new Response(child.stdout).text(); + expect(await child.exited).toBe(0); + expect(attempt).toMatch(/^[a-f0-9]{32}$/); + let recovered = false; + const started = performance.now(); + await expectClosedRefusal( + nativeProjectDown({ + ...opts, + recover: true, + invoke: async (request) => { + if (request.args[1] === "foreground-owner") { + return ownerSelection("exited"); + } + if (request.args[1] === "recover-live-owner") { + recovered = true; + return { + run: run.run, + recovered: true, + data_retained: true, + publisher_retired: true, + }; + } + return snapshot(recovered); + }, + }), + `hack restart --recover-frontend --expect-finalization-attempt ${attempt}` + ); + expect(recovered).toBe(true); + // An exited frontend is not awaited for the ordinary finalization budget. + expect(performance.now() - started).toBeLessThan(10_000); + expect(await loadNativeProjectRun(opts.scope)).toEqual(run); +}); diff --git a/tests/native-project-inspect.test.ts b/tests/native-project-inspect.test.ts index ff7c5b4f6..af1e906e6 100644 --- a/tests/native-project-inspect.test.ts +++ b/tests/native-project-inspect.test.ts @@ -1,5 +1,9 @@ import { expect, test } from "bun:test"; -import { inspectNativeProjectGraph } from "../src/backends/native-project-inspect.ts"; +import { + INSPECT_BUSY_WAIT_MS, + inspectNativeProjectGraph, + OBSERVER_BUSY_WAIT_MS, +} from "../src/backends/native-project-inspect.ts"; const opts = { runtime: { binary: "/unused", home: "/unused" }, @@ -37,3 +41,53 @@ test("unknown inspection failures are never retried", async () => { ).rejects.toThrow("unknown"); expect(calls).toBe(1); }); +test("inspections inside mutating commands outlast contention that ends an observer's budget", async () => { + const busy = () => { + throw new Error( + "Native runtime request failed (provider_busy); inspect owned state before retrying." + ); + }; + // A holder that keeps the lease longer than the ps/logs budget. + const contended = (until: number) => async () => { + if (performance.now() < until) { + busy(); + } + return { ready: true }; + }; + const started = performance.now(); + expect( + await inspectNativeProjectGraph({ + ...opts, + invoke: contended(started + OBSERVER_BUSY_WAIT_MS + 300), + }) + ).toEqual({ ready: true }); + expect(performance.now() - started).toBeGreaterThan(OBSERVER_BUSY_WAIT_MS); + const observer = performance.now(); + await expect( + inspectNativeProjectGraph({ + ...opts, + busyWaitMs: OBSERVER_BUSY_WAIT_MS, + invoke: contended(observer + OBSERVER_BUSY_WAIT_MS + 1000), + }) + ).rejects.toThrow("provider_busy"); + expect(INSPECT_BUSY_WAIT_MS).toBeGreaterThan(OBSERVER_BUSY_WAIT_MS); +}, 15_000); +test("an aborted caller stops waiting out lease contention", async () => { + const controller = new AbortController(); + let calls = 0; + const pending = inspectNativeProjectGraph({ + ...opts, + signal: controller.signal, + invoke: async () => { + calls++; + if (calls === 2) { + controller.abort(); + } + throw new Error( + "Native runtime request failed (provider_busy); inspect owned state before retrying." + ); + }, + }); + await expect(pending).rejects.toThrow("provider_busy"); + expect(calls).toBe(2); +}); diff --git a/tests/native-project-recovery.test.ts b/tests/native-project-recovery.test.ts index 4bf9ae250..ccdda426e 100644 --- a/tests/native-project-recovery.test.ts +++ b/tests/native-project-recovery.test.ts @@ -6,6 +6,7 @@ import { mkdtemp, realpath, rm, + symlink, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -183,6 +184,29 @@ test("recovery requires stopped owned graph, vacant HTTPS, and empty lifecycle", expect(f.events).toEqual(["processes", "graph", "authority", "port"]); }); +test("recovery accepts a project that never recorded lifecycle processes, but not an unsafe record", async () => { + const f = await fixture(); + // Absent state reads as no entries, as the lifecycle runtime treats it. + await rm(f.lifecycle, { recursive: true }); + await verifyNativeFrontendRecovery(f.opts); + expect(f.events).toEqual(["processes", "graph", "authority", "port"]); + // A state path that is not a regular private file still refuses. + await mkdir(join(f.lifecycle, "state.json"), { recursive: true }); + await expectClosedRefusal( + verifyNativeFrontendRecovery(f.opts), + "cannot prove" + ); + await rm(join(f.lifecycle, "state.json"), { recursive: true }); + await symlink( + "/nonexistent-lifecycle-state", + join(f.lifecycle, "state.json") + ); + await expectClosedRefusal( + verifyNativeFrontendRecovery(f.opts), + "cannot prove" + ); +}); + test("recovery cleans owned lifecycle only after graph, authority and port proof", async () => { const f = await fixture(); await writeFile(join(f.lifecycle, "state.json"), '{"entries":[{}]}'); From c46d34e0dda5c61a97cd934e3476757edb46e7a3 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:22:02 -0500 Subject: [PATCH 2/3] fix(native): resume an interrupted down --recover; gate Linux-dead lock helper `hack down --recover` refused a retry after an interruption past the runtime's cleanup intent: the receipt was no longer `ready-observed`, so the guard reported an owner that exited before its start was confirmed and stranded the partially stopped graph and its stale publication. `graph foreground-owner` now also returns `live_owner_recovery`, the original receipt digest of a same-boot dead-owner recovery whose intent exists but whose cleanup or publisher retirement is unfinished. `down --recover` resumes that recovery by its own selection, then takes the stopped path (host processes, owed journaled hooks, finalization) without replaying a stop. An exit before restore admission, with no recovery intent, still refuses. `Lock::acquire_existing_waiting` is used only by macOS code and is now gated like its callers, fixing the Linux clippy dead-code error. Refs HACK-1954 Co-Authored-By: Claude Opus 5.5 --- .../src/provider/graph/live_owner_cleanup.rs | 78 +++++++++++++++++++ .../runtime-core/src/provider/graph/mod.rs | 12 ++- .../src/provider/graph/retained_startup.rs | 8 +- packages/runtime-core/src/provider/state.rs | 2 + src/backends/native-project-down.ts | 43 +++++++++- tests/native-project-down.test.ts | 60 ++++++++++++++ 6 files changed, 197 insertions(+), 6 deletions(-) diff --git a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs index a6b620b37..ee8aa8161 100644 --- a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs @@ -34,6 +34,33 @@ pub(super) fn current_completion( Ok(intent.complete_sha256.as_deref() == Some(complete.as_str())) } +/// The original receipt digest of a same-boot recovery whose durable intent exists but +/// whose recovery has not finished (cleanup still pending, or the stale publication not +/// yet retired), so an interrupted `hack down --recover` resumes it by its own selection. +/// Selection only: `recover_live_owner` revalidates the intent against current evidence. +pub(super) fn resumable_selection( + root: &Path, + receipt: &Receipt, + publication_present: bool, +) -> Result, CandidateError> { + if !exists(&root.join(FILE))? { + return Ok(None); + } + let intent: Intent = state::read(&root.join(FILE))?; + let current = digest(receipt)?; + Ok(unfinished( + intent.complete_sha256.as_deref(), + ¤t, + publication_present, + ) + .then_some(intent.original_sha256)) +} +/// Cleanup not yet complete, or complete for this exact receipt with the stale +/// publication still present. A completed recovery of an older receipt is history. +fn unfinished(complete: Option<&str>, current: &str, publication_present: bool) -> bool { + complete.is_none_or(|complete| publication_present && complete == current) +} + /// Select an independent data-removal request from the exact completed same-boot /// generation. A retry pins the entire committed proof, not just receipt phases. pub(super) fn removal_selection( @@ -649,6 +676,20 @@ pub(super) fn retire( )) } +#[cfg(test)] +mod resumable_tests { + #[test] + fn only_an_unfinished_recovery_is_resumed() { + // Interrupted after the cleanup intent: resumed whatever the receipt shows. + assert!(super::unfinished(None, "a", true)); + assert!(super::unfinished(None, "a", false)); + // Cleanup completed for this receipt but the publication was not retired. + assert!(super::unfinished(Some("a"), "a", true)); + // Fully retired, or a completed recovery of an older generation: history. + assert!(!super::unfinished(Some("a"), "a", false)); + assert!(!super::unfinished(Some("a"), "b", true)); + } +} #[cfg(test)] mod tests { use super::*; @@ -811,6 +852,43 @@ mod tests { ); value } + #[test] + fn an_interrupted_recovery_is_selected_by_its_original_receipt_until_retired() { + let fixture = super::super::tests::Fixture::new(); + let mut current = receipt(); + // No recovery intent: nothing to resume (an unadmitted exit stays refused). + assert_eq!( + resumable_selection(&fixture.0, ¤t, true).unwrap(), + None + ); + let mut intent = completed(receipt()); + intent.original_sha256 = "1".repeat(64); + intent.complete_sha256 = None; + state::write(&fixture.0.join(FILE), &intent).unwrap(); + current.phase = "cleanup-intent".into(); + assert_eq!( + resumable_selection(&fixture.0, ¤t, true).unwrap(), + Some("1".repeat(64)) + ); + current.phase = "stopped-data-retained".into(); + intent.complete_sha256 = Some(digest(¤t).unwrap()); + fs::remove_file(fixture.0.join(FILE)).unwrap(); + state::write(&fixture.0.join(FILE), &intent).unwrap(); + assert_eq!( + resumable_selection(&fixture.0, ¤t, true).unwrap(), + Some("1".repeat(64)) + ); + assert_eq!( + resumable_selection(&fixture.0, ¤t, false).unwrap(), + None + ); + current.owner = "9".repeat(32); + assert_eq!( + resumable_selection(&fixture.0, ¤t, true).unwrap(), + None + ); + } + fn completed(original: Receipt) -> Intent { let mut stopped = original.clone(); stopped.phase = "stopped-data-retained".into(); diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 74f625ce8..ff361b6eb 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -1669,8 +1669,10 @@ fn endpoint_network<'a>( } /// Read-only selection for an explicit dead-owner recovery such as `hack down --recover`: -/// the run's receipt phase, the SHA-256 of its exact receipt bytes and what its -/// foreground publication shows (`absent`, `running`, `exited` or `uncertain`). +/// the run's receipt phase, the SHA-256 of its exact receipt bytes, what its +/// foreground publication shows (`absent`, `running`, `exited` or `uncertain`) and, +/// for an interrupted same-boot dead-owner recovery, the original receipt digest that +/// resumes it (`live_owner_recovery`). /// /// It takes no lease, contacts no guest and grants no authority. The digest only lets /// a caller select the receipt it observed; each recovery command revalidates its own @@ -1682,6 +1684,11 @@ pub fn foreground_owner(candidate: &Candidate, run: &str) -> Result Result CandidateError { _ => refused(), } } +/// A ready graph, or a same-boot dead-owner recovery of one that was interrupted: +/// both are stopped by `hack down --recover`. fn exited_ready(candidate: &Candidate, run: &str) -> bool { Owner::load(candidate) .ok() .and_then(|owner| load_at(directory(candidate, run).ok()?, run, &owner.token).ok()) - .is_some_and(|(receipt, _)| receipt.phase == "ready-observed") + .is_some_and(|(receipt, root)| { + receipt.phase == "ready-observed" + || super::live_owner_cleanup::resumable_selection(&root, &receipt, true) + .is_ok_and(|selected| selected.is_some()) + }) } fn no_file(root: &Path, name: &str) -> Result<(), CandidateError> { diff --git a/packages/runtime-core/src/provider/state.rs b/packages/runtime-core/src/provider/state.rs index 1c795fdca..57144b5d0 100644 --- a/packages/runtime-core/src/provider/state.rs +++ b/packages/runtime-core/src/provider/state.rs @@ -110,6 +110,7 @@ impl Lock { Self::from_file(file) } + #[cfg(target_os = "macos")] /// [`Self::acquire_existing`], waiting up to `wait` for another holder, such as the /// short read-only inspection behind `hack ps`, to release the lock. Only acquisition /// is retried: nothing is created or admitted while waiting, and on expiry the @@ -821,6 +822,7 @@ mod tests { assert!(Lock::acquire(&root).is_ok()); fs::remove_dir_all(root).unwrap(); } + #[cfg(target_os = "macos")] #[test] fn waiting_existing_acquisition_outlasts_a_brief_holder_and_refuses_a_lasting_one() { let root = fixture(); diff --git a/src/backends/native-project-down.ts b/src/backends/native-project-down.ts index 048ab9c7c..bf41c0a85 100644 --- a/src/backends/native-project-down.ts +++ b/src/backends/native-project-down.ts @@ -105,6 +105,8 @@ type ForegroundOwnerSelection = { readonly phase: string; readonly receiptSha256: string; readonly foreground: "absent" | "running" | "exited" | "uncertain"; + /** Original receipt digest of an interrupted same-boot dead-owner recovery. */ + readonly liveOwnerRecovery: string | null; }; function ownerRecoveryRefused(): Error { @@ -134,7 +136,12 @@ async function selectForegroundOwner(opts: { typeof value.receipt_sha256 !== "string" || !RECEIPT_SHA.test(value.receipt_sha256) || typeof value.foreground !== "string" || - !FOREGROUND_STATES.has(value.foreground) + !FOREGROUND_STATES.has(value.foreground) || + !( + value.live_owner_recovery === null || + (typeof value.live_owner_recovery === "string" && + RECEIPT_SHA.test(value.live_owner_recovery)) + ) ) { throw ownerRecoveryRefused(); } @@ -142,6 +149,7 @@ async function selectForegroundOwner(opts: { phase: value.phase, receiptSha256: value.receipt_sha256, foreground: value.foreground as ForegroundOwnerSelection["foreground"], + liveOwnerRecovery: value.live_owner_recovery, }; } @@ -183,6 +191,17 @@ async function requestOwnedStop(opts: { } return; } + await recoverLiveOwner({ ...opts, expectReceipt: selection.receiptSha256 }); +} + +/** Same-boot dead-owner recovery of the exact selected receipt; it resumes its own intent. */ +async function recoverLiveOwner(opts: { + readonly runtime: NativeRuntimeSelection; + readonly projectRoot: string; + readonly run: NativeProjectRun; + readonly invoke: typeof invokeNativeRuntime; + readonly expectReceipt: string; +}): Promise { const result = await opts.invoke({ runtime: opts.runtime, cwd: opts.projectRoot, @@ -192,7 +211,7 @@ async function requestOwnedStop(opts: { "--run-id", opts.run.run, "--expect-receipt", - selection.receiptSha256, + opts.expectReceipt, "--json", ], timeoutMs: 590_000, @@ -215,6 +234,11 @@ function requireRecoverableOwner(selection: ForegroundOwnerSelection): void { "Native down --recover found a running foreground owner for this run. Stop it with Ctrl-C in its terminal, or run hack down without --recover. Nothing was changed." ); } + // An interrupted recovery resumes by its own selection, whatever the receipt + // phase or partial publication retirement it left behind. + if (selection.liveOwnerRecovery !== null) { + return; + } if (selection.foreground === "uncertain") { throw ownerRecoveryRefused(); } @@ -286,7 +310,7 @@ export async function nativeProjectDown(opts: { run: run.run, invoke, }); - const initial = await inspect(); + let initial = await inspect(); verify(initial, run, false); const selection = opts.recover ? await selectForegroundOwner({ @@ -299,6 +323,19 @@ export async function nativeProjectDown(opts: { if (selection) { requireRecoverableOwner(selection); } + if (selection?.liveOwnerRecovery) { + // Finish the interrupted recovery first; the stopped path below then retires + // host processes and runs only journaled owed hooks, never replaying a stop. + await recoverLiveOwner({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + run, + invoke, + expectReceipt: selection.liveOwnerRecovery, + }); + initial = await inspect(); + verify(initial, run, false); + } const finalization = opts.deferFinalization ? undefined : await captureFinalization({ scope: opts.scope, run }); diff --git a/tests/native-project-down.test.ts b/tests/native-project-down.test.ts index 4f17eed1c..4af712070 100644 --- a/tests/native-project-down.test.ts +++ b/tests/native-project-down.test.ts @@ -1061,6 +1061,7 @@ function ownerSelection(foreground: string, phase = "ready-observed") { phase, receipt_sha256: receiptSha, foreground, + live_owner_recovery: null as string | null, }; } @@ -1204,6 +1205,11 @@ test.each([ selection: { ...ownerSelection("exited"), owner: "9".repeat(32) }, text: "cannot confirm this run's foreground owner state", }, + { + name: "a malformed recovery selection", + selection: { ...ownerSelection("exited"), live_owner_recovery: "short" }, + text: "cannot confirm this run's foreground owner state", + }, { name: "a malformed receipt digest", selection: { ...ownerSelection("exited"), receipt_sha256: "short" }, @@ -1287,3 +1293,57 @@ test("down --recover names the exact frontend recovery when the owning frontend expect(performance.now() - started).toBeLessThan(10_000); expect(await loadNativeProjectRun(opts.scope)).toEqual(run); }); + +test("down --recover resumes an interrupted dead-owner recovery by its original receipt", async () => { + const opts = await fixture(); + const original = "1".repeat(64); + const events: string[] = []; + let recovered = false; + const result = await nativeProjectDown({ + ...opts, + recover: true, + before: async () => { + events.push("before"); + }, + retireHostProcesses: async () => { + events.push("retire-host"); + }, + invoke: async (request) => { + events.push(request.args[1]!); + if (request.args[1] === "foreground-owner") { + // Interrupted after its cleanup intent: the receipt is no longer ready. + return { + ...ownerSelection("exited", "cleanup-intent"), + live_owner_recovery: original, + }; + } + if (request.args[1] === "recover-live-owner") { + expect(request.args).toContain(original); + expect(request.args).not.toContain(receiptSha); + recovered = true; + return { + run: run.run, + phase: "stopped-data-retained", + recovered: true, + data_retained: true, + publisher_retired: true, + }; + } + const value = snapshot(recovered); + if (!recovered) { + value.receipt.phase = "cleanup-intent"; + } + return value; + }, + }); + expect(result.status).toBe("stopped"); + // The resumed stop runs no before hook and requests no ordinary cleanup. + expect(events).toEqual([ + "inspect", + "foreground-owner", + "recover-live-owner", + "inspect", + "retire-host", + ]); + expect(await loadNativeProjectRun(opts.scope)).toEqual(run); +}); From 0d8d9135dd20cc6eb2b82f96c1dc99a94042d290 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:33:07 -0500 Subject: [PATCH 3/3] fix(native): keep a recovery resumable until its reservation is released An interrupted `hack down --recover` could stop after both foreground publisher paths were retired but before `finish_retirement` released the run's dependency reservation. The resume selection then treated the recovery as finished because the publication was absent, so a retry took the already-stopped path and left the reservation behind; the next start of a run with dependency bindings refused because a reservation for the same run already existed. The selection now stays resumable while either retirement effect remains: the stale publication or the run's dependency reservation record. A retry therefore re-enters `recover-live-owner`, whose completed-intent path finishes retirement and releases the reservation. Refs HACK-1954 Co-Authored-By: Claude Opus 5.5 --- .../src/provider/graph/dependency_slots.rs | 10 +++ .../src/provider/graph/live_owner_cleanup.rs | 67 ++++++++++--------- .../runtime-core/src/provider/graph/mod.rs | 1 + .../src/provider/graph/retained_startup.rs | 2 +- 4 files changed, 47 insertions(+), 33 deletions(-) diff --git a/packages/runtime-core/src/provider/graph/dependency_slots.rs b/packages/runtime-core/src/provider/graph/dependency_slots.rs index b73112bf4..626e88210 100644 --- a/packages/runtime-core/src/provider/graph/dependency_slots.rs +++ b/packages/runtime-core/src/provider/graph/dependency_slots.rs @@ -41,6 +41,16 @@ fn refused() -> CandidateError { fn root(candidate: &Candidate) -> PathBuf { candidate.state_root.join("run/dependency-assignments") } +/// Whether a dependency reservation record exists for this run. Observation only. +pub(super) fn reservation_recorded( + candidate: &Candidate, + run: &str, +) -> Result { + if !hex(run, 32) { + return Err(refused()); + } + Ok(!absent(&root(candidate).join(format!("{run}.json")))?) +} fn socket(home: &Path, slot: u8) -> PathBuf { home.join(format!("dependency-{slot:02}.sock")) } diff --git a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs index ee8aa8161..1dce732eb 100644 --- a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs @@ -35,10 +35,13 @@ pub(super) fn current_completion( } /// The original receipt digest of a same-boot recovery whose durable intent exists but -/// whose recovery has not finished (cleanup still pending, or the stale publication not -/// yet retired), so an interrupted `hack down --recover` resumes it by its own selection. +/// whose recovery has not finished, so an interrupted `hack down --recover` resumes it +/// by its own selection. Unfinished means cleanup is still pending, or cleanup completed +/// for this exact receipt while a retirement effect of `finish_retirement` remains: the +/// stale publication, or this run's dependency reservation (released last). /// Selection only: `recover_live_owner` revalidates the intent against current evidence. pub(super) fn resumable_selection( + candidate: &Candidate, root: &Path, receipt: &Receipt, publication_present: bool, @@ -48,17 +51,17 @@ pub(super) fn resumable_selection( } let intent: Intent = state::read(&root.join(FILE))?; let current = digest(receipt)?; - Ok(unfinished( - intent.complete_sha256.as_deref(), - ¤t, - publication_present, + let remaining = + publication_present || dependency_slots::reservation_recorded(candidate, &receipt.run)?; + Ok( + unfinished(intent.complete_sha256.as_deref(), ¤t, remaining) + .then_some(intent.original_sha256), ) - .then_some(intent.original_sha256)) } -/// Cleanup not yet complete, or complete for this exact receipt with the stale -/// publication still present. A completed recovery of an older receipt is history. -fn unfinished(complete: Option<&str>, current: &str, publication_present: bool) -> bool { - complete.is_none_or(|complete| publication_present && complete == current) +/// Cleanup not yet complete, or complete for this exact receipt with a retirement effect +/// remaining. A completed recovery of an older receipt is history. +fn unfinished(complete: Option<&str>, current: &str, retirement_remaining: bool) -> bool { + complete.is_none_or(|complete| retirement_remaining && complete == current) } /// Select an independent data-removal request from the exact completed same-boot @@ -683,7 +686,7 @@ mod resumable_tests { // Interrupted after the cleanup intent: resumed whatever the receipt shows. assert!(super::unfinished(None, "a", true)); assert!(super::unfinished(None, "a", false)); - // Cleanup completed for this receipt but the publication was not retired. + // Cleanup completed for this receipt but a retirement effect remains. assert!(super::unfinished(Some("a"), "a", true)); // Fully retired, or a completed recovery of an older generation: history. assert!(!super::unfinished(Some("a"), "a", false)); @@ -855,38 +858,38 @@ mod tests { #[test] fn an_interrupted_recovery_is_selected_by_its_original_receipt_until_retired() { let fixture = super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + let select = |receipt: &Receipt, publication: bool| { + resumable_selection(&candidate, &fixture.0, receipt, publication).unwrap() + }; let mut current = receipt(); // No recovery intent: nothing to resume (an unadmitted exit stays refused). - assert_eq!( - resumable_selection(&fixture.0, ¤t, true).unwrap(), - None - ); + assert_eq!(select(¤t, true), None); let mut intent = completed(receipt()); intent.original_sha256 = "1".repeat(64); intent.complete_sha256 = None; state::write(&fixture.0.join(FILE), &intent).unwrap(); + // Interrupted after the cleanup intent. current.phase = "cleanup-intent".into(); - assert_eq!( - resumable_selection(&fixture.0, ¤t, true).unwrap(), - Some("1".repeat(64)) - ); + assert_eq!(select(¤t, true), Some("1".repeat(64))); current.phase = "stopped-data-retained".into(); intent.complete_sha256 = Some(digest(¤t).unwrap()); fs::remove_file(fixture.0.join(FILE)).unwrap(); state::write(&fixture.0.join(FILE), &intent).unwrap(); - assert_eq!( - resumable_selection(&fixture.0, ¤t, true).unwrap(), - Some("1".repeat(64)) - ); - assert_eq!( - resumable_selection(&fixture.0, ¤t, false).unwrap(), - None - ); + // Interrupted after cleanup, before publisher retirement. + assert_eq!(select(¤t, true), Some("1".repeat(64))); + // Interrupted after both publisher paths were retired but before the + // dependency reservation was released: still resumed by its receipt. + let reservations = candidate.state_root.join("run/dependency-assignments"); + state::private_directory(&reservations).unwrap(); + let record = reservations.join(format!("{}.json", current.run)); + fs::write(&record, b"{}").unwrap(); + assert_eq!(select(¤t, false), Some("1".repeat(64))); + // Every retirement effect done: history, not a resumable recovery. + fs::remove_file(&record).unwrap(); + assert_eq!(select(¤t, false), None); current.owner = "9".repeat(32); - assert_eq!( - resumable_selection(&fixture.0, ¤t, true).unwrap(), - None - ); + assert_eq!(select(¤t, true), None); } fn completed(original: Receipt) -> Intent { diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index ff361b6eb..897965a10 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -1685,6 +1685,7 @@ pub fn foreground_owner(candidate: &Candidate, run: &str) -> Result bool { .and_then(|owner| load_at(directory(candidate, run).ok()?, run, &owner.token).ok()) .is_some_and(|(receipt, root)| { receipt.phase == "ready-observed" - || super::live_owner_cleanup::resumable_selection(&root, &receipt, true) + || super::live_owner_cleanup::resumable_selection(candidate, &root, &receipt, true) .is_ok_and(|selected| selected.is_some()) }) }