Keep KVM sandbox clocks synchronized with the host

.github/workflows/ci.yml

@@ -37,6 +37,9 @@ jobs:
       - name: KVM guest resolver handoff
         run: tests/kvm_guest_dns.sh
 
+      - name: KVM guest clock lifecycle
+        run: bash tests/guest_clock_lifecycle.sh
+
       - name: Sandbox-runner liveness checks
         run: tests/sandbox_runner_healthcheck.sh
 

README.md

@@ -75,6 +75,23 @@ Two modes are supported:
 - **NsJail mode** (`kvmEnabled: false`): Direct NsJail sandboxing with Linux namespaces and cgroups
 - **MicroVM mode** (`kvmEnabled: true`): libkrun microVM with its own kernel, NsJail runs inside the guest
 
+### KVM guest time
+
+Linux KVM guests use chrony and the native `/dev/ptp0` KVM virtual PTP clock to
+keep their wall clock aligned with the host. Without continuous correction,
+clock-rate differences can eventually make fresh execution manifests appear
+to come from the future. Changing the guest clocksource or restarting the
+runner does not provide ongoing synchronization.
+
+The KVM bootstrap waits for synchronization before starting the API, then
+supervises both processes: losing the clock daemon stops the guest workload.
+Only the first clock update may step time; subsequent corrections slew it.
+The guest kernel must expose a working, UTC host-referenced KVM PTP device;
+boot fails closed if synchronization is unavailable. Clock control stays in a
+root-only guest-private `/run` directory, with no network NTP source, new host
+capability, or change to user-job clock restrictions or manifest tolerance.
+Direct NsJail, local OCI/macOS and Lambda startup do not run this guest daemon.
+
 ## Remote stateful environments
 
 The `remote-bridge` backend keeps the Code API as the policy and queue boundary

api/Dockerfile

@@ -98,6 +98,7 @@ RUN apt-get update && \
     bsdextrautils \
     locales \
     procps \
+    chrony \
     iptables \
     bc \
     file \
@@ -168,6 +169,7 @@ RUN rm -f /usr/bin/nsenter /usr/bin/unshare /usr/bin/chroot /usr/sbin/chroot \
 
 COPY api/src/entrypoint.sh ./entrypoint.sh
 COPY api/src/guest-dns.sh ./guest-dns.sh
+COPY api/src/guest-clock.sh ./guest-clock.sh
 COPY api/src/hosted-app-launcher.sh /usr/local/bin/codeapi-hosted-app-launcher
 RUN chmod +x ./entrypoint.sh /usr/local/bin/codeapi-hosted-app-launcher
 

api/config/guest-chrony.conf

@@ -0,0 +1,16 @@
+# Chrony configuration for CodeAPI KVM microVM guest
+# Synchronizes guest CLOCK_REALTIME directly from host via KVM virtual PTP hardware clock (/dev/ptp0).
+# The KVM virtual PTP device provides host CLOCK_REALTIME (UTC, no TAI offset).
+refclock PHC /dev/ptp0 poll 0 dpoll -2
+
+# Only the first update may step: the API starts after synchronization.
+# Subsequent corrections slew the clock rather than jumping active jobs.
+makestep 0.1 1
+
+# Disable network NTP server and monitoring ports; operate strictly in refclock mode.
+port 0
+cmdport 0
+
+# Restrict control interface to a private Unix domain socket and fixed pidfile.
+bindcmdaddress /run/codeapi-clock/chronyd.sock
+pidfile /run/codeapi-clock/chronyd.pid

api/src/guest-clock.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+# KVM microVM guest clock synchronization wrapper.
+# Synchronizes guest system CLOCK_REALTIME directly from the host via the KVM
+# virtual PTP hardware clock (/dev/ptp0) using chrony, before starting the API.
+# The host clock is the root authority; no external network NTP traffic is used.
+# Runtime state is isolated in a private, root-owned directory under /run.
+
+# Own the bootstrap shell: libkrun signals this PID, not a supervisor subshell.
+# Like the previous exec path, this function exits rather than returning.
+run_clock_synchronized_guest() {
+    set -euo pipefail
+    local root="${1?root argument required (empty for the guest)}"
+    shift
+    [ "$#" -gt 0 ] || { echo 'ERROR: guest command required' >&2; exit 1; }
+    export PATH="${PATH:-/usr/local/bin:/usr/bin:/bin}:/usr/sbin"
+
+    local clock_dir="$root/run/codeapi-clock"
+    local socket_file="$clock_dir/chronyd.sock"
+    local chrony_pid="" guest_pid="" sync_pid=""
+
+    # /run is a fresh guest-private tmpfs. mkdir refuses existing entries,
+    # including dangling symlinks; sandbox UIDs cannot replace its children.
+    (umask 077; mkdir "$clock_dir") || exit 1
+
+    stop_child() {
+        local pid="$1"
+        [ -n "$pid" ] || return 0
+        kill -TERM "$pid" 2>/dev/null || true
+        local attempt
+        for attempt in {1..20}; do
+            kill -0 "$pid" 2>/dev/null || break
+            sleep 0.05
+        done
+        kill -KILL "$pid" 2>/dev/null || true
+        wait "$pid" 2>/dev/null || true
+    }
+
+    cleanup() {
+        trap - EXIT INT TERM
+        stop_child "$sync_pid"
+        stop_child "$guest_pid"
+        stop_child "$chrony_pid"
+        rm -rf -- "$clock_dir"
+    }
+    trap cleanup EXIT
+    trap 'exit 130' INT
+    trap 'exit 143' TERM
+
+    chronyd -d -u root -f "$root/sandbox_api/config/guest-chrony.conf" &
+    chrony_pid=$!
+
+    # Allow socket creation before chronyc connects; refuse a dead daemon.
+    local attempt
+    for attempt in {1..100}; do
+        kill -0 "$chrony_pid" 2>/dev/null || {
+            echo 'ERROR: guest clock daemon exited during startup' >&2
+            exit 1
+        }
+        [ ! -e "$socket_file" ] || break
+        sleep 0.05
+    done
+    [ -e "$socket_file" ] || {
+        echo 'ERROR: guest clock control socket did not become ready' >&2
+        exit 1
+    }
+
+    # Bound both clock acquisition and a stuck control client. Backgrounding
+    # the wait lets INT/TERM interrupt startup and clean up the whole group.
+    timeout -k 1 15 chronyc -h "$socket_file" waitsync 100 0.05 0 0.1 &
+    sync_pid=$!
+    local status=0
+    wait "$sync_pid" || status=$?
+    sync_pid=""
+    if [ "$status" -ne 0 ] || ! kill -0 "$chrony_pid" 2>/dev/null; then
+        echo 'ERROR: guest clock synchronization failed' >&2
+        exit 1
+    fi
+
+    "$@" <&0 &
+    guest_pid=$!
+    local finished=""
+    status=0
+    wait -n -p finished "$chrony_pid" "$guest_pid" 2>/dev/null || status=$?
+    if [ "${finished:-}" = "$chrony_pid" ]; then
+        chrony_pid=""
+        echo 'ERROR: guest clock daemon exited; stopping the guest command' >&2
+        exit 1
+    fi
+    if [ "${finished:-}" = "$guest_pid" ]; then
+        guest_pid=""
+        exit "$status"
+    fi
+
+    # Bash can cache a very short-lived child's exit before wait -n starts.
+    # Read its status once without swallowing failures or spinning on ECHILD.
+    kill -0 "$chrony_pid" 2>/dev/null || exit 1
+    status=0
+    wait "$guest_pid" || status=$?
+    guest_pid=""
+    exit "$status"
+}
+

api/src/guest-dns.sh

@@ -41,7 +41,8 @@ run_guest_command() {
     # DNS separate from /tmp, which the normal API entrypoint mounts later.
     mount -t tmpfs -o size=1m,mode=0755 tmpfs "$root/run" || return 1
     configure_guest_dns "$root" || return 1
-    exec -- "$@"
+    source "$(dirname "${BASH_SOURCE[0]}")/guest-clock.sh"
+    run_clock_synchronized_guest "$root" "$@"
 }
 
 if [ "${BASH_SOURCE[0]}" = "$0" ]; then

docker/Dockerfile.worker-sandbox

@@ -130,6 +130,7 @@ RUN apt-install \
     bsdextrautils \
     locales \
     procps \
+    chrony \
     iptables \
     bc \
     jq \
@@ -170,6 +171,7 @@ COPY --from=sandbox-builder /app/.build ./.build
 COPY api/config ./config
 COPY api/src/entrypoint.sh ./entrypoint.sh
 COPY api/src/guest-dns.sh ./guest-dns.sh
+COPY api/src/guest-clock.sh ./guest-clock.sh
 RUN chmod +x ./entrypoint.sh
 
 RUN mkdir -p /pkgs /tmp/sandbox

launcher/Dockerfile

@@ -60,6 +60,7 @@ RUN apt-get update && \
     bsdextrautils \
     locales \
     procps \
+    chrony \
     iptables \
     socat \
     libnl-route-3-200 \
@@ -110,6 +111,7 @@ RUN rm -f /usr/bin/nsenter /usr/bin/unshare /usr/bin/chroot /usr/sbin/chroot \
 
 COPY api/src/entrypoint.sh ./entrypoint.sh
 COPY api/src/guest-dns.sh ./guest-dns.sh
+COPY api/src/guest-clock.sh ./guest-clock.sh
 RUN chmod +x ./entrypoint.sh
 
 # ============================================================================