fix(native): keep the VM responsive during image builds and stop a silent guest

packages/runtime-core/README.md

@@ -490,7 +490,12 @@ that does not name an owned network or `none`. Cold builds need free guest stora
 cache to 8 GiB, and `runtime reclaim-build-images` removes only owned, superseded,
 unreferenced images (`provider/image_build/reclaim.rs`; model
 `tests/models/tla/native-build-retention`). Native `up` calls the build operation
-for Compose `build:` services only when `HACK_NATIVE_BUILDS=1` is set.
+for Compose `build:` services only when `HACK_NATIVE_BUILDS=1` is set. The client reaches
+the engine through a private relay (`provider/image_build/client_relay.rs`): the pinned
+provider's macOS socket forwarding can loop forever when a host client closes a forwarded
+connection while the guest is still sending, which saturated the guest and stopped every later
+agent and engine request. The relay always drains guest bytes, forwards a client's end of
+stream only as a half-close, and closes a provider connection only after the guest ended it.
 
 Source binaries embed their build checkout. If a copied Cargo target directory
 retains a different checkout identity, use a fresh `--target-dir` for checks and
@@ -500,6 +505,12 @@ tests; Cargo reporting a cached binary as fresh does not qualify that identity.
 native process/disk identity, guest protocol, and lifecycle decisions. Guest requests use a bounded
 connection to the existing socket; they cannot silently start or recover a VM. Failed operations
 retain their phase and receipts. Recovery preserves disks and labels an unclean exit explicitly.
+When the guest agent stops responding (three pings over about 13 seconds start no reply),
+`runtime down` and `runtime recover` stop this home's recorded provider after verifying its PID,
+start time, user and executable, without guest quiescence, and record `recovered-unclean`; a
+guest that answers keeps the normal clean stop, and `recover` still refuses it. Both refuse with
+`provider_busy` while a native image build holds its fence; finishing or canceling that build
+(Ctrl-C its `hack up`) releases the fence, also when a build process was killed.
 
 SmolVM seeds a machine's disks during its first start (`machine create` makes none) from expanded
 `*-template.ext4` files that it creates from the verified compressed templates and then reuses.

packages/runtime-core/src/provider/agent.rs

@@ -112,6 +112,59 @@ fn exchange(
     serde_json::from_slice(&body).map_err(|_| failure("Malformed guest response."))
 }
 
+/// Whether the agent at `path` starts any reply to a ping within `timeout`. No
+/// connection, a connection that ends before a reply, and no reply by the deadline are
+/// silence; any reply byte, even of an unexpected response, is an answer. Only a stop of
+/// a guest that stopped responding relies on this.
+pub(super) fn answers_ping(path: &Path, timeout: Duration) -> bool {
+    let deadline = Instant::now() + timeout;
+    let Ok(bytes) = encode(&json!({"method":"ping"})) else {
+        return false;
+    };
+    let Ok(mut stream) = UnixStream::connect(path) else {
+        return false;
+    };
+    let mut frame = (bytes.len() as u32).to_be_bytes().to_vec();
+    frame.extend_from_slice(&bytes);
+    if stream.set_nonblocking(true).is_err() {
+        return false;
+    }
+    let mut written = 0;
+    let mut reply = [0_u8; 1];
+    loop {
+        let Some(remaining) = deadline.checked_duration_since(Instant::now()) else {
+            return false;
+        };
+        if written < frame.len() {
+            match stream.write(&frame[written..]) {
+                Ok(count) => written += count,
+                Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {}
+                Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {}
+                Err(_) => return false,
+            }
+        }
+        match stream.read(&mut reply) {
+            Ok(0) => return false,
+            Ok(_) => return true,
+            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {}
+            Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {}
+            Err(_) => return false,
+        }
+        let mut descriptor = libc::pollfd {
+            fd: stream.as_raw_fd(),
+            events: if written < frame.len() {
+                libc::POLLIN | libc::POLLOUT
+            } else {
+                libc::POLLIN
+            },
+            revents: 0,
+        };
+        let milliseconds = (remaining.as_millis() + 1).min(i32::MAX as u128) as i32;
+        // SAFETY: the descriptor is borrowed from the live stream for this synchronous wait.
+        unsafe { libc::poll(&mut descriptor, 1, milliseconds) };
+    }
+}
+
 pub fn ping(path: &Path) -> Result<(), CandidateError> {
     let response = request(path, json!({"method":"ping"}), Duration::from_secs(3))?;
     if response["status"] != "pong" || response["version"] != 1 {

packages/runtime-core/src/provider/image_build.rs

@@ -2,7 +2,9 @@
 //!
 //! One operation builds one image for one build key inside the guest engine, through
 //! the pinned host BuildKit client (`docker-buildx`, docker driver) talking to the
-//! engine's verified forwarded socket. The built image lands in the guest image store,
+//! engine's verified forwarded socket through a private relay ([`client_relay`]) that
+//! keeps the provider's socket forwarding out of a known busy loop. The built image
+//! lands in the guest image store,
 //! where graph admission already looks, and the caller pins the service to the content
 //! ID it returns, so review, receipts and restore treat it like a pulled image.
 //!
@@ -68,6 +70,7 @@ use std::{
     time::{Duration, Instant},
 };
 
+mod client_relay;
 mod output;
 pub mod reclaim;
 #[cfg(test)]
@@ -875,8 +878,17 @@ pub(super) fn run<P: BuildPool>(
     // outlives the client run.
     let _definition = RemoveOnDrop(scratch.join("definition").join(BAKE_DEFINITION));
     let _policy = RemoveOnDrop(scratch.join("definition").join(SOURCE_POLICY));
+    // The client reaches the engine only through the relay, which keeps the provider's
+    // forwarded socket out of the state described in `client_relay`.
+    let relay = client_relay::ClientRelay::start(&identity.socket)?;
     let mut client = command(
-        &builder, &scratch, &metadata, inputs, &identity, &pins, &expected,
+        &builder,
+        &scratch,
+        &metadata,
+        inputs,
+        relay.socket(),
+        &pins,
+        &expected,
     )?;
     inherit_locks(&mut client, &[&fence, &key_lock]);
     let ended = supervise(
@@ -886,6 +898,8 @@ pub(super) fn run<P: BuildPool>(
         output::Filter::new(inputs.args.values().map(String::as_str)),
         pool.output(),
     );
+    // The client has exited or was stopped; wait for the guest to end its connections.
+    relay.finish(client_relay::FINISH_GRACE);
     let ended = match ended {
         Ok(ended) => ended,
         Err(error) => {
@@ -973,6 +987,20 @@ pub(super) fn run<P: BuildPool>(
     })
 }
 
+/// Hold the shared build fence as an admitted build does, for the macOS-only lifecycle
+/// test of the silent-guest stop.
+#[cfg(all(test, target_os = "macos"))]
+pub(super) fn hold_build_fence_for_test(candidate: &Candidate) -> impl Drop {
+    let root = builds_root(candidate);
+    state::private_directory(&root).unwrap();
+    Flock::try_lock(
+        Flock::open(&root.join("active.lock")).unwrap(),
+        libc::LOCK_SH,
+    )
+    .unwrap()
+    .unwrap()
+}
+
 /// The image config digest the client reports for the built image.
 fn built_image_id(metadata: &Path) -> Result<String, CandidateError> {
     let bytes = read_bounded_file(metadata, MAX_METADATA_BYTES)?.ok_or_else(|| {
@@ -1101,7 +1129,7 @@ fn command(
     scratch: &Path,
     metadata: &Path,
     inputs: &Inputs,
-    identity: &PoolIdentity,
+    engine: &Path,
     pins: &BTreeMap<String, String>,
     labels: &BTreeMap<&'static str, String>,
 ) -> Result<std::process::Command, CandidateError> {
@@ -1128,10 +1156,7 @@ fn command(
         .current_dir(scratch)
         .env("HOME", scratch)
         .env("TMPDIR", scratch)
-        .env(
-            "DOCKER_HOST",
-            format!("unix://{}", identity.socket.display()),
-        )
+        .env("DOCKER_HOST", format!("unix://{}", engine.display()))
         // An empty private client configuration: no contexts, credential helpers or
         // proxies from the user's Docker setup.
         .env("DOCKER_CONFIG", &config)