diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index ded85be95..a4404fa78 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -662,6 +662,30 @@ Provider setup is separate and explicit. Obtain the exact archives identified in ./hack-native --candidate-root /absolute/private/candidate-home runtime probe --json ``` +Preparation keeps each archive it extracted (`providers/smolvm-installing/release.tar.gz` +and `providers/docker-installing/release.tgz`). If something outside Hack changes a +prepared tree, for example a cleanup or backup tool that prunes empty directories, +every start refuses with `artifact_digest_mismatch` and the message names the repair. +A stopped runtime's `runtime status` and `runtime down` still work, because they run +nothing from the tree, so an installer quiescence check is not blocked. To repair the +tree, stop the runtime and run: + +```sh +./hack-native --candidate-root /absolute/private/candidate-home runtime repair-provider-tree --json +``` + +The repair holds the provider lock and requires a stopped (or uninitialized) runtime. +It requires the preparation record to match this executor's pins and the kept archive +to be an owned regular file with the pinned SHA-256. It then copies and hashes the +archive again, extracts it, checks the code signature (SmolVM), and requires the fresh +tree to match the digest recorded at preparation. Only then does it rename the changed +tree aside as `providers/-changed-` and put the fresh tree in its place. It +keeps the changed tree for inspection and never deletes it; remove it yourself once +you no longer need it. Runtime state, disks and volumes under `run/` are not touched. +An intact tree is reported with `repaired: false`. A repair that fails a check changes +no tree and keeps its private `-repairing-` staging directory. Without a kept archive the repair +refuses (`provider_repair_unavailable`); prepare a fresh candidate home instead. + The pins also name a BuildKit client (`buildx`, `darwin-arm64`) for native image builds. Prepare it before using `HACK_NATIVE_BUILDS=1`; it is optional otherwise: diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index 0f3de3e59..8e6270a98 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -117,6 +117,7 @@ Usage: hack-local runtime up --profile development --internet --json hack-local runtime network extend --allow-host [--allow-host ] --json hack-local runtime up|status|down|recover [--json] + hack-local runtime repair-provider-tree [--json] hack-local runtime host-filesystem-recovery [--json] hack-local runtime recover-host-filesystem --expect-sha256 --accept-legacy-device-rebind [--json] hack-local node serve|status|inspect @@ -1119,6 +1120,11 @@ fn run() -> Result<(), CandidateError> { hash, )?)?; } + ["runtime", "repair-provider-tree"] | ["runtime", "repair-provider-tree", "--json"] => { + print_json(&hack_runtime_core::provider::repair_provider_trees( + &discover_candidate(&requested)?, + )?)?; + } ["runtime", action] | ["runtime", action, "--json"] => { let candidate = discover_candidate(&requested)?; let result = match *action { diff --git a/packages/runtime-core/src/provider/artifact.rs b/packages/runtime-core/src/provider/artifact.rs index b68405f09..31d772111 100644 --- a/packages/runtime-core/src/provider/artifact.rs +++ b/packages/runtime-core/src/provider/artifact.rs @@ -175,16 +175,23 @@ pub fn prepare(candidate: &Candidate, archive: &Path) -> Result Result { - reject_aliased_state(&root(candidate))?; - let artifact: Artifact = state::read(&candidate.state_root.join("providers/smolvm.json"))?; - if artifact.version != VERSION - || artifact.archive_sha256 != ARCHIVE_SHA256 - || artifact.tree_sha256 != tree_digest(&root(candidate))? + verify_tree(candidate, &SMOLVM) +} + +/// Verify a prepared tree against its record before anything executes from it. +pub(super) fn verify_tree( + candidate: &Candidate, + pin: &TreePin, +) -> Result { + let providers = candidate.state_root.join("providers"); + let prepared = providers.join(pin.directory); + reject_aliased_state(&prepared)?; + let artifact: Artifact = state::read(&providers.join(pin.record))?; + if artifact.version != pin.version + || artifact.archive_sha256 != pin.archive_sha256 + || artifact.tree_sha256 != tree_digest(&prepared)? { - return Err(CandidateError::new( - "artifact_digest_mismatch", - "Prepared SmolVM tree changed; refusing execution.", - )); + return Err(CandidateError::new("artifact_digest_mismatch", pin.changed)); } Ok(artifact) } @@ -247,18 +254,248 @@ pub fn prepare_engine(candidate: &Candidate, archive: &Path) -> Result Result { - reject_aliased_state(&engine_root(candidate))?; - let artifact: Artifact = state::read(&candidate.state_root.join("providers/docker.json"))?; - if artifact.version != ENGINE_VERSION - || artifact.archive_sha256 != ENGINE_SHA256 - || artifact.tree_sha256 != tree_digest(&engine_root(candidate))? - { - return Err(CandidateError::new( + verify_tree(candidate, &ENGINE) +} + +/// A provider tree that preparation extracted from a pinned archive and whose archive +/// it retained in its staging directory, so the tree can be re-prepared later. +pub(super) struct TreePin { + label: &'static str, + version: &'static str, + archive_sha256: &'static str, + /// The prepared tree under `providers/`. + directory: &'static str, + /// The archive's top-level directory as extracted. + extracted: &'static str, + /// The tree's preparation record under `providers/`. + record: &'static str, + /// Preparation's staging directory under `providers/`; it retains the archive. + staging: &'static str, + archive: &'static str, + /// The executable whose code signature preparation checks, if any. + signed: Option<&'static str>, + /// The fixed refusal for a changed tree; it names the repair. + changed: &'static str, +} + +impl TreePin { + pub(super) fn record_name(&self) -> &'static str { + self.record + } +} + +pub(super) const SMOLVM: TreePin = TreePin { + label: "SmolVM", + version: VERSION, + archive_sha256: ARCHIVE_SHA256, + directory: DIRECTORY, + extracted: DIRECTORY, + record: "smolvm.json", + staging: "smolvm-installing", + archive: "release.tar.gz", + signed: Some("smolvm-bin"), + changed: "Prepared SmolVM tree changed; refusing execution. With the runtime stopped, run runtime repair-provider-tree to re-prepare it from the verified retained archive; runtime state and volumes are kept.", +}; + +pub(super) const ENGINE: TreePin = TreePin { + label: "Docker engine", + version: ENGINE_VERSION, + archive_sha256: ENGINE_SHA256, + directory: "docker-29.5.2-linux-arm64", + extracted: "docker", + record: "docker.json", + staging: "docker-installing", + archive: "release.tgz", + signed: None, + changed: "Prepared Docker engine changed; refusing execution. With the runtime stopped, run runtime repair-provider-tree to re-prepare it from the verified retained archive; runtime state and volumes are kept.", +}; + +/// The outcome of re-preparing one prepared provider tree. +#[derive(Debug, Serialize)] +pub struct TreeRepair { + pub provider: &'static str, + pub version: String, + pub tree_sha256: String, + /// `false` when the prepared tree already matched its record; nothing changed. + pub repaired: bool, + /// The changed tree's new name under `providers/`. It is retained, never deleted. + pub retained_changed_tree: Option, +} + +fn repair_refused(code: &'static str, message: &'static str) -> CandidateError { + CandidateError::new(code, message) +} + +/// The archive preparation retained, when it is still an owned regular file in an +/// owned directory that no other user can write. +fn retained_archive(providers: &Path, pin: &TreePin) -> Result { + let staging = providers.join(pin.staging); + let archive = staging.join(pin.archive); + // SAFETY: geteuid has no preconditions. + let uid = unsafe { libc::geteuid() }; + for (path, directory) in [(&staging, true), (&archive, false)] { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(repair_refused( + "provider_repair_unavailable", + "No retained provider archive to repair from; prepare a fresh candidate home.", + )); + } + Err(error) => return Err(io(error)), + }; + let kind = if directory { + metadata.is_dir() + } else { + metadata.is_file() && metadata.nlink() == 1 + }; + if !kind || metadata.uid() != uid || metadata.mode() & 0o022 != 0 { + return Err(repair_refused( + "foreign_state", + "Retained provider archive is not an owned regular file in an owned directory.", + )); + } + } + Ok(archive) +} + +fn nonce() -> Result { + let mut random = [0_u8; 8]; + File::open("/dev/urandom") + .map_err(io)? + .read_exact(&mut random) + .map_err(io)?; + Ok(random.iter().map(|b| format!("{b:02x}")).collect()) +} + +fn sync_directory(path: &Path) -> Result<(), CandidateError> { + File::open(path).map_err(io)?.sync_all().map_err(io) +} + +/// Re-prepare one prepared provider tree that something outside Hack changed. +/// +/// The caller holds the provider lock with the runtime stopped, so nothing executes +/// from the tree. The repair re-proves the tree from scratch: the preparation record +/// must match this executor's pin, the retained archive must be an owned regular file +/// whose SHA-256 is the pin, its owned copy is hashed again before extraction, and the +/// fresh extraction (signature checked, as preparation does) must hash to the recorded +/// tree digest. Only then is the changed tree renamed aside, retained and never +/// deleted, and the fresh tree renamed into place. Runtime state, disks and volumes +/// live under `run/` and are untouched. A failed repair keeps its private staging. +/// +/// `None` means the tree was never prepared; an intact tree is reported unchanged. +pub(super) fn repair_tree( + candidate: &Candidate, + pin: &TreePin, + _stopped: &state::Lock, +) -> Result, CandidateError> { + let providers = candidate.state_root.join("providers"); + let record = providers.join(pin.record); + if !record.try_exists().map_err(io)? { + return Ok(None); + } + state::check_private_directory(&providers)?; + let recorded: Artifact = state::read(&record)?; + if recorded.version != pin.version || recorded.archive_sha256 != pin.archive_sha256 { + return Err(repair_refused( "artifact_digest_mismatch", - "Prepared Docker engine changed; refusing execution.", + "Provider preparation record does not match this executor's pin; repair refused.", )); } - Ok(artifact) + let prepared = providers.join(pin.directory); + let present = match fs::symlink_metadata(&prepared) { + Ok(metadata) if metadata.is_dir() => true, + Ok(_) => { + return Err(repair_refused( + "foreign_state", + "Prepared provider tree is not a directory; repair refused.", + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => return Err(io(error)), + }; + let report = |repaired, retained_changed_tree| TreeRepair { + provider: pin.label, + version: recorded.version.clone(), + tree_sha256: recorded.tree_sha256.clone(), + repaired, + retained_changed_tree, + }; + if present { + reject_aliased_state(&prepared)?; + // A tree that cannot even be hashed (for example a new special file) is changed. + if tree_digest(&prepared).ok().as_deref() == Some(recorded.tree_sha256.as_str()) { + return Ok(Some(report(false, None))); + } + } + let archive = retained_archive(&providers, pin)?; + if digest(&archive)? != pin.archive_sha256 { + return Err(repair_refused( + "artifact_digest_mismatch", + "Retained provider archive does not match the pinned SHA-256; repair refused.", + )); + } + let nonce = nonce()?; + let work = providers.join(format!("{}-repairing-{nonce}", pin.directory)); + fs::DirBuilder::new() + .mode(0o700) + .create(&work) + .map_err(io)?; + let owned = work.join(pin.archive); + let mut source = File::open(&archive).map_err(io)?; + let mut destination = fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(&owned) + .map_err(io)?; + std::io::copy(&mut source, &mut destination).map_err(io)?; + destination.sync_all().map_err(io)?; + drop(destination); + // Rehash the owned copy to close replacement between the check and extraction. + if digest(&owned)? != pin.archive_sha256 { + return Err(repair_refused( + "artifact_digest_mismatch", + "Retained provider archive changed during repair; repair staging retained.", + )); + } + run( + clean_command(Path::new("/usr/bin/tar")) + .arg("-xzf") + .arg(&owned) + .arg("-C") + .arg(&work), + Duration::from_secs(60), + )?; + let extracted = work.join(pin.extracted); + if let Some(executable) = pin.signed { + run( + clean_command(Path::new("/usr/bin/codesign")) + .args(["--verify", "--strict"]) + .arg(extracted.join(executable)), + Duration::from_secs(10), + )?; + } + if tree_digest(&extracted)? != recorded.tree_sha256 { + return Err(repair_refused( + "artifact_digest_mismatch", + "Re-extracted provider tree does not match its recorded digest; repair staging retained.", + )); + } + let retained = if present { + let name = format!("{}-changed-{nonce}", pin.directory); + fs::rename(&prepared, providers.join(&name)).map_err(io)?; + Some(name) + } else { + None + }; + fs::rename(&extracted, &prepared).map_err(io)?; + sync_directory(&providers)?; + // Only this repair's owned archive copy and its now-empty directory; never recursive. + let _ = fs::remove_file(&owned); + let _ = fs::remove_dir(&work); + verify_tree(candidate, pin)?; + Ok(Some(report(true, retained))) } /// Pinned standalone BuildKit client for native image builds. @@ -411,6 +648,125 @@ fn verify_binary(candidate: &Candidate, pin: &BinaryPin) -> Result TreePin { + let source = scratch.join("archive-source"); + populate(&source.join(TEST_DIRECTORY)); + let archive = scratch.join("fixture-release.tar.gz"); + tar(&[ + "-czf".as_ref(), + archive.as_os_str(), + "-C".as_ref(), + source.as_os_str(), + TEST_DIRECTORY.as_ref(), + ]); + let sha256: &'static str = Box::leak(digest(&archive).unwrap().into_boxed_str()); + let pin = TreePin { + label: "Test tree", + version: "1.0.0", + archive_sha256: sha256, + directory: TEST_DIRECTORY, + extracted: TEST_DIRECTORY, + record: "test-tree.json", + staging: "test-tree-installing", + archive: "release.tar.gz", + signed: None, + changed: "Prepared test tree changed; refusing execution.", + }; + let providers = candidate.state_root.join("providers"); + state::private_directory(&providers).unwrap(); + let staging = providers.join(pin.staging); + fs::create_dir(&staging).unwrap(); + let retained = staging.join(pin.archive); + fs::copy(&archive, &retained).unwrap(); + tar(&[ + "-xzf".as_ref(), + retained.as_os_str(), + "-C".as_ref(), + staging.as_os_str(), + ]); + let extracted = staging.join(pin.extracted); + let artifact = Artifact { + version: pin.version.into(), + archive_sha256: sha256.into(), + tree_sha256: tree_digest(&extracted).unwrap(), + }; + fs::rename(extracted, providers.join(pin.directory)).unwrap(); + state::write(&providers.join(pin.record), &artifact).unwrap(); + verify_tree(candidate, &pin).unwrap(); + pin + } + + /// A SmolVM tree and record under the real pins, without an archive: enough for + /// `verify` to pass until the tree changes. Only the macOS lifecycle tests use it. + #[cfg(target_os = "macos")] + pub(in crate::provider) fn prepared_smolvm(candidate: &Candidate) { + let providers = candidate.state_root.join("providers"); + state::private_directory(&providers).unwrap(); + populate(&root(candidate)); + let artifact = Artifact { + version: VERSION.into(), + archive_sha256: ARCHIVE_SHA256.into(), + tree_sha256: tree_digest(&root(candidate)).unwrap(), + }; + state::write(&providers.join(SMOLVM.record), &artifact).unwrap(); + verify(candidate).unwrap(); + } + + /// Remove every empty directory below `path`, deepest first, as a cleanup tool does. + pub(in crate::provider) fn prune_empty_directories(path: &Path) -> usize { + let mut removed = 0; + for entry in fs::read_dir(path).unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + removed += prune_empty_directories(&entry.path()); + if fs::read_dir(entry.path()).unwrap().next().is_none() { + fs::remove_dir(entry.path()).unwrap(); + removed += 1; + } + } + } + removed + } +} + #[cfg(test)] mod tests { use super::*; @@ -428,6 +784,136 @@ mod tests { fs::create_dir(&path).unwrap(); path } + fn providers(candidate: &Candidate) -> PathBuf { + candidate.state_root.join("providers") + } + fn lease(candidate: &Candidate) -> state::Lock { + state::Lock::acquire(&candidate.state_root.join("run/smolvm")).unwrap() + } + fn repair_staging(candidate: &Candidate) -> Vec { + fs::read_dir(providers(candidate)) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name.contains("-repairing-")) + .collect() + } + + #[test] + fn a_tree_pruned_of_empty_directories_is_repaired_from_its_retained_archive() { + let root = fixture(); + let candidate = Candidate::discover(&root).unwrap(); + let pin = tree_fixture::prepare(&candidate, &root); + let prepared = providers(&candidate).join(pin.directory); + let record: Artifact = state::read(&providers(&candidate).join(pin.record)).unwrap(); + assert!(tree_fixture::prune_empty_directories(&prepared) >= 6); + assert_eq!( + verify_tree(&candidate, &pin).unwrap_err().code, + "artifact_digest_mismatch" + ); + + let lease = lease(&candidate); + let repaired = repair_tree(&candidate, &pin, &lease).unwrap().unwrap(); + assert!(repaired.repaired); + assert_eq!(repaired.tree_sha256, record.tree_sha256); + verify_tree(&candidate, &pin).unwrap(); + assert!(prepared.join("agent-rootfs/etc/periodic/daily").is_dir()); + // The changed tree is retained aside, never deleted. + let retained = providers(&candidate).join(repaired.retained_changed_tree.unwrap()); + assert!(retained.join("README.txt").is_file()); + assert!(!retained.join("agent-rootfs/dev").exists()); + // The retained archive stays; this repair's own staging is gone. + assert!( + providers(&candidate) + .join(pin.staging) + .join(pin.archive) + .is_file() + ); + assert!(repair_staging(&candidate).is_empty()); + let after: Artifact = state::read(&providers(&candidate).join(pin.record)).unwrap(); + assert_eq!(after.tree_sha256, record.tree_sha256); + + // An intact tree is reported, not replaced. + let identity = fs::metadata(&prepared).unwrap().ino(); + let again = repair_tree(&candidate, &pin, &lease).unwrap().unwrap(); + assert!(!again.repaired); + assert!(again.retained_changed_tree.is_none()); + assert_eq!(fs::metadata(&prepared).unwrap().ino(), identity); + drop(lease); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn repair_refuses_without_a_verified_archive_or_a_matching_extraction() { + let root = fixture(); + let candidate = Candidate::discover(&root).unwrap(); + // A tree without a preparation record was never prepared: nothing to repair. + let unprepared = tree_fixture::prepare(&candidate, &root); + fs::remove_file(providers(&candidate).join(unprepared.record)).unwrap(); + let first = lease(&candidate); + assert!( + repair_tree(&candidate, &unprepared, &first) + .unwrap() + .is_none() + ); + drop(first); + fs::remove_dir_all(&candidate.state_root).unwrap(); + fs::remove_dir_all(root.join("archive-source")).unwrap(); + + let pin = tree_fixture::prepare(&candidate, &root); + let lease = lease(&candidate); + let prepared = providers(&candidate).join(pin.directory); + let retained = providers(&candidate).join(pin.staging).join(pin.archive); + tree_fixture::prune_empty_directories(&prepared); + let changed = tree_digest(&prepared).unwrap(); + + fs::write(&retained, b"not the pinned archive").unwrap(); + assert_eq!( + repair_tree(&candidate, &pin, &lease).unwrap_err().code, + "artifact_digest_mismatch" + ); + fs::remove_file(&retained).unwrap(); + assert_eq!( + repair_tree(&candidate, &pin, &lease).unwrap_err().code, + "provider_repair_unavailable" + ); + std::os::unix::fs::symlink(root.join("fixture-release.tar.gz"), &retained).unwrap(); + assert_eq!( + repair_tree(&candidate, &pin, &lease).unwrap_err().code, + "foreign_state" + ); + fs::remove_file(&retained).unwrap(); + fs::copy(root.join("fixture-release.tar.gz"), &retained).unwrap(); + assert!(repair_staging(&candidate).is_empty()); + + // A record no extraction can match keeps the changed tree and the repair staging. + let record = providers(&candidate).join(pin.record); + let mut recorded: Artifact = state::read(&record).unwrap(); + let original = recorded.tree_sha256.clone(); + recorded.tree_sha256 = "0".repeat(64); + fs::remove_file(&record).unwrap(); + state::write(&record, &recorded).unwrap(); + assert_eq!( + repair_tree(&candidate, &pin, &lease).unwrap_err().code, + "artifact_digest_mismatch" + ); + assert_eq!(tree_digest(&prepared).unwrap(), changed); + assert_eq!(repair_staging(&candidate).len(), 1); + + // A record for another version is refused before any extraction. + recorded.tree_sha256 = original; + recorded.version = "1.0.1".into(); + fs::remove_file(&record).unwrap(); + state::write(&record, &recorded).unwrap(); + assert_eq!( + repair_tree(&candidate, &pin, &lease).unwrap_err().code, + "artifact_digest_mismatch" + ); + assert_eq!(repair_staging(&candidate).len(), 1); + assert_eq!(tree_digest(&prepared).unwrap(), changed); + drop(lease); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn published_pin_manifest_matches_the_executor() { let pins: serde_json::Value = diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index ff3ddcb5b..77127d8f5 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -8,6 +8,8 @@ mod prepared_boot; mod pressure_tests; #[cfg(any(target_os = "macos", test))] mod private_child; +#[cfg(all(test, target_os = "macos"))] +mod provider_repair_tests; mod relay_process; mod short_home; use super::{ @@ -1989,30 +1991,40 @@ fn engine_ping(path: &Path) -> Result<(), CandidateError> { Ok(()) } +/// Pool phases written only before a start or after proven provider absence. +const STOPPED_PHASES: [&str; 4] = [ + "stopped", + "stopped-before-engine", + "stopped-after-engine-failure", + "recovered-unclean", +]; + +/// A stopped phase with no live provider; no recorded provider (`None`) is as stopped +/// as a dead one. Nothing runs from the prepared provider tree in this state. +fn provider_stopped(phase: &str, process_alive: Option) -> bool { + STOPPED_PHASES.contains(&phase) && process_alive != Some(true) +} + pub fn down(candidate: &Candidate) -> Result { // Status is read-only and idempotent down on an uninitialized pool creates nothing. let initial = status(candidate)?; if initial.phase == "uninitialized" { return Ok(initial); } - artifact::verify(candidate)?; let _lock = state::Lock::acquire(&root(candidate))?; // Builds take their shared fence only under this lease, so none can start after this. super::image_build::require_no_active_build(candidate)?; let mut owner = Owner::load(candidate)?; + let stopped = provider_stopped(&owner.phase, initial.process_alive); + // Only stopping a provider reaches the prepared tree. A stopped pool's down executes + // nothing from it, so a tree changed outside Hack cannot block quiescence; any later + // start still verifies the tree (see `repair_provider_trees`). + if !stopped { + artifact::verify(candidate)?; + } super::hostname_authority::managed::stop(candidate, &owner)?; super::publication::release(candidate, &owner.token, None)?; - if [ - "stopped", - "stopped-before-engine", - "stopped-after-engine-failure", - "recovered-unclean", - ] - .contains(&owner.phase.as_str()) - && initial.process_alive != Some(true) - { - // These phases are written only before a start or after proven absence; no recorded - // provider (`None`) is as stopped as a dead one. + if stopped { return status(candidate); } verify_live(candidate, &owner)?; @@ -2028,6 +2040,63 @@ pub fn down(candidate: &Candidate) -> Result { } } +/// The outcome of `runtime repair-provider-tree`. +#[derive(Serialize)] +pub struct ProviderRepair { + /// Each prepared provider tree, repaired or found intact. + pub trees: Vec, + pub runtime: RuntimeStatus, +} + +/// Re-prepare provider trees that something outside Hack changed (for example a cleanup +/// tool that pruned empty directories), from the archives preparation retained. +/// +/// Requires a stopped or uninitialized pool under the provider lock, so no provider runs +/// from a tree while it is replaced and no start can begin meanwhile. Each tree is +/// re-proved from its pinned archive digest and recorded tree digest; see +/// `artifact::repair_tree`. Runtime state, disks and volumes are untouched. Until a +/// repair succeeds, every start keeps refusing the changed tree. +pub fn repair_provider_trees(candidate: &Candidate) -> Result { + repair_trees(candidate, &[artifact::SMOLVM, artifact::ENGINE]) +} + +fn repair_trees( + candidate: &Candidate, + pins: &[artifact::TreePin], +) -> Result { + let providers = candidate.state_root.join("providers"); + let mut any = false; + for pin in pins { + any |= providers.join(pin.record_name()).try_exists().map_err(io)?; + } + if !any { + // Nothing was prepared: create no state, like status and down. + return Ok(ProviderRepair { + trees: Vec::new(), + runtime: status(candidate)?, + }); + } + let lock = state::Lock::acquire(&root(candidate))?; + super::image_build::require_no_active_build(candidate)?; + let current = status(candidate)?; + if current.phase != "uninitialized" && !provider_stopped(¤t.phase, current.process_alive) + { + return Err(CandidateError::new( + "provider_repair_requires_stopped_runtime", + "Provider tree repair requires a stopped runtime; run runtime down first.", + )); + } + let mut trees = Vec::new(); + for pin in pins { + lock.verify_path(&root(candidate))?; + trees.extend(artifact::repair_tree(candidate, pin, &lock)?); + } + Ok(ProviderRepair { + trees, + runtime: status(candidate)?, + }) +} + /// How many pings, a few seconds apart, a guest agent must leave unanswered before /// `down` or `recover` treats it as stopped responding. const SILENT_PINGS: u32 = 3; diff --git a/packages/runtime-core/src/provider/lifecycle/provider_repair_tests.rs b/packages/runtime-core/src/provider/lifecycle/provider_repair_tests.rs new file mode 100644 index 000000000..9e0a49e48 --- /dev/null +++ b/packages/runtime-core/src/provider/lifecycle/provider_repair_tests.rs @@ -0,0 +1,140 @@ +//! HACK-1950: a prepared provider tree changed outside Hack (here, every empty directory +//! pruned) must not block a stopped pool's `status` and `down`, and a supported repair +//! re-prepares it from the retained archive while execution stays refused until then. +use super::*; +use crate::provider::artifact::tree_fixture; +use crate::provider::{NetworkIntent, Profile}; +use std::path::PathBuf; + +/// A stopped (or otherwise recorded) pool with no live provider. Its short HOME alias +/// lives under `/private/tmp` for the socket path budget and is removed on drop. +struct Pool { + root: PathBuf, + candidate: Candidate, + owner: Owner, +} + +impl Pool { + fn new(phase: &str) -> Self { + let mut random = [0_u8; 8]; + File::open("/dev/urandom") + .unwrap() + .read_exact(&mut random) + .unwrap(); + let nonce: String = random.iter().map(|b| format!("{b:02x}")).collect(); + let root = PathBuf::from("/private/tmp").join(format!("hkr-{nonce}")); + fs::create_dir(&root).unwrap(); + let candidate = Candidate::discover(&root).unwrap(); + let mut owner = + Owner::create(&candidate, Profile::Research, None, NetworkIntent::Isolated).unwrap(); + let data = owner.real_data_dir(&candidate).unwrap(); + state::private_directory(&data).unwrap(); + fs::write(data.join("name"), &owner.machine).unwrap(); + owner.phase = phase.into(); + owner.save(&candidate).unwrap(); + Self { + root, + candidate, + owner, + } + } +} + +impl Drop for Pool { + fn drop(&mut self) { + let _ = fs::remove_file(&self.owner.short_home); + fs::remove_dir_all(&self.root).unwrap(); + } +} + +#[test] +fn a_stopped_pool_with_a_pruned_provider_tree_goes_down_and_is_repaired() { + let pool = Pool::new("stopped"); + let candidate = &pool.candidate; + tree_fixture::prepared_smolvm(candidate); + let pin = tree_fixture::prepare(candidate, &pool.root); + let providers = candidate.state_root.join("providers"); + assert!(tree_fixture::prune_empty_directories(&artifact::root(candidate)) >= 6); + assert!(tree_fixture::prune_empty_directories(&providers.join("test-tree-1.0.0")) >= 6); + assert_eq!( + artifact::verify(candidate).unwrap_err().code, + "artifact_digest_mismatch" + ); + + // The installer's quiescence check: status, down, status, all without the tree. + for observed in [status(candidate), down(candidate), status(candidate)] { + let observed = observed.unwrap(); + assert_eq!(observed.phase, "stopped"); + assert_eq!(observed.process_alive, None); + } + // Down executed nothing from the tree; execution is still refused. + assert_eq!( + artifact::verify(candidate).unwrap_err().code, + "artifact_digest_mismatch" + ); + assert!( + artifact::verify(candidate) + .unwrap_err() + .message + .contains("runtime repair-provider-tree") + ); + + let repaired = repair_trees(candidate, &[pin]).unwrap(); + assert_eq!(repaired.runtime.phase, "stopped"); + assert_eq!(repaired.trees.len(), 1); + assert!(repaired.trees[0].repaired); + assert!(repaired.trees[0].retained_changed_tree.is_some()); + // Runtime state is untouched: the same owner and machine remain. + let after = Owner::load(candidate).unwrap(); + assert_eq!(after.token, pool.owner.token); + assert_eq!(after.machine, pool.owner.machine); + assert_eq!(after.phase, "stopped"); + // The pinned SmolVM tree has no retained archive here, so its repair is unavailable + // and the real executor keeps refusing it. + assert_eq!( + repair_provider_trees(candidate).err().unwrap().code, + "provider_repair_unavailable" + ); + assert_eq!( + artifact::verify(candidate).unwrap_err().code, + "artifact_digest_mismatch" + ); +} + +#[test] +fn a_pool_that_is_not_proven_stopped_keeps_verifying_and_refuses_repair() { + let pool = Pool::new("running"); + let candidate = &pool.candidate; + tree_fixture::prepared_smolvm(candidate); + let pin = tree_fixture::prepare(candidate, &pool.root); + tree_fixture::prune_empty_directories(&artifact::root(candidate)); + tree_fixture::prune_empty_directories(&candidate.state_root.join("providers/test-tree-1.0.0")); + // Stopping an unproven provider reaches the tree, so down still verifies it first. + assert_eq!( + down(candidate).err().unwrap().code, + "artifact_digest_mismatch" + ); + assert_eq!( + repair_trees(candidate, &[pin]).err().unwrap().code, + "provider_repair_requires_stopped_runtime" + ); + assert_eq!(Owner::load(candidate).unwrap().phase, "running"); +} + +#[test] +fn repair_of_a_home_without_prepared_providers_creates_no_state() { + let mut random = [0_u8; 8]; + File::open("/dev/urandom") + .unwrap() + .read_exact(&mut random) + .unwrap(); + let nonce: String = random.iter().map(|b| format!("{b:02x}")).collect(); + let root = PathBuf::from("/private/tmp").join(format!("hkr-{nonce}")); + fs::create_dir(&root).unwrap(); + let candidate = Candidate::discover(&root).unwrap(); + let repaired = repair_provider_trees(&candidate).unwrap(); + assert!(repaired.trees.is_empty()); + assert_eq!(repaired.runtime.phase, "uninitialized"); + assert!(!candidate.state_root.exists()); + fs::remove_dir_all(root).unwrap(); +} diff --git a/packages/runtime-core/src/provider/mod.rs b/packages/runtime-core/src/provider/mod.rs index c6333e5b8..2ce6b242d 100644 --- a/packages/runtime-core/src/provider/mod.rs +++ b/packages/runtime-core/src/provider/mod.rs @@ -132,9 +132,9 @@ mod socket_requests; mod state; pub use lifecycle::{ PreparedBaseBuilt, build_prepared_base, check_project_share, down, prepared_base_status, - probe_with_profile, recover, remove_prepared_base, status, up, up_with_bridge, - up_with_capabilities, up_with_minimum_bridges, up_with_network_sockets, up_with_prepared_base, - up_with_profile, up_with_project_share, up_with_retained_project_share, + probe_with_profile, recover, remove_prepared_base, repair_provider_trees, status, up, + up_with_bridge, up_with_capabilities, up_with_minimum_bridges, up_with_network_sockets, + up_with_prepared_base, up_with_profile, up_with_project_share, up_with_retained_project_share, up_with_socket_requests, up_with_sockets, verify_prepared_base, }; pub use socket_requests::SocketRequests;