From 89ca42102ae4376ae89efc22af2ebca545f8e8c8 Mon Sep 17 00:00:00 2001 From: Duy Nguyen <57234880+duynguyen158@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:16:48 -0500 Subject: [PATCH] [fix] synchronize KVM guest clocks with the host --- .github/workflows/ci.yml | 3 + README.md | 17 ++ api/Dockerfile | 2 + api/config/guest-chrony.conf | 16 ++ api/src/guest-clock.sh | 102 ++++++++ api/src/guest-dns.sh | 3 +- docker/Dockerfile.worker-sandbox | 2 + launcher/Dockerfile | 2 + tests/guest_clock_lifecycle.sh | 419 +++++++++++++++++++++++++++++++ tests/kvm_guest_dns.sh | 26 ++ 10 files changed, 591 insertions(+), 1 deletion(-) create mode 100644 api/config/guest-chrony.conf create mode 100644 api/src/guest-clock.sh create mode 100644 tests/guest_clock_lifecycle.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7337e5a2..eb993b00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,9 @@ jobs: - name: KVM guest resolver handoff run: tests/kvm_guest_dns.sh + - name: KVM guest clock lifecycle + run: bash tests/guest_clock_lifecycle.sh + - name: Sandbox-runner liveness checks run: tests/sandbox_runner_healthcheck.sh diff --git a/README.md b/README.md index 94ad3b9c..ae67341d 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,23 @@ Two modes are supported: - **NsJail mode** (`kvmEnabled: false`): Direct NsJail sandboxing with Linux namespaces and cgroups - **MicroVM mode** (`kvmEnabled: true`): libkrun microVM with its own kernel, NsJail runs inside the guest +### KVM guest time + +Linux KVM guests use chrony and the native `/dev/ptp0` KVM virtual PTP clock to +keep their wall clock aligned with the host. Without continuous correction, +clock-rate differences can eventually make fresh execution manifests appear +to come from the future. Changing the guest clocksource or restarting the +runner does not provide ongoing synchronization. + +The KVM bootstrap waits for synchronization before starting the API, then +supervises both processes: losing the clock daemon stops the guest workload. +Only the first clock update may step time; subsequent corrections slew it. +The guest kernel must expose a working, UTC host-referenced KVM PTP device; +boot fails closed if synchronization is unavailable. Clock control stays in a +root-only guest-private `/run` directory, with no network NTP source, new host +capability, or change to user-job clock restrictions or manifest tolerance. +Direct NsJail, local OCI/macOS and Lambda startup do not run this guest daemon. + ## Remote stateful environments The `remote-bridge` backend keeps the Code API as the policy and queue boundary diff --git a/api/Dockerfile b/api/Dockerfile index e7dbcc4e..ec573325 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -98,6 +98,7 @@ RUN apt-get update && \ bsdextrautils \ locales \ procps \ + chrony \ iptables \ bc \ file \ @@ -168,6 +169,7 @@ RUN rm -f /usr/bin/nsenter /usr/bin/unshare /usr/bin/chroot /usr/sbin/chroot \ COPY api/src/entrypoint.sh ./entrypoint.sh COPY api/src/guest-dns.sh ./guest-dns.sh +COPY api/src/guest-clock.sh ./guest-clock.sh COPY api/src/hosted-app-launcher.sh /usr/local/bin/codeapi-hosted-app-launcher RUN chmod +x ./entrypoint.sh /usr/local/bin/codeapi-hosted-app-launcher diff --git a/api/config/guest-chrony.conf b/api/config/guest-chrony.conf new file mode 100644 index 00000000..6de6a60c --- /dev/null +++ b/api/config/guest-chrony.conf @@ -0,0 +1,16 @@ +# Chrony configuration for CodeAPI KVM microVM guest +# Synchronizes guest CLOCK_REALTIME directly from host via KVM virtual PTP hardware clock (/dev/ptp0). +# The KVM virtual PTP device provides host CLOCK_REALTIME (UTC, no TAI offset). +refclock PHC /dev/ptp0 poll 0 dpoll -2 + +# Only the first update may step: the API starts after synchronization. +# Subsequent corrections slew the clock rather than jumping active jobs. +makestep 0.1 1 + +# Disable network NTP server and monitoring ports; operate strictly in refclock mode. +port 0 +cmdport 0 + +# Restrict control interface to a private Unix domain socket and fixed pidfile. +bindcmdaddress /run/codeapi-clock/chronyd.sock +pidfile /run/codeapi-clock/chronyd.pid diff --git a/api/src/guest-clock.sh b/api/src/guest-clock.sh new file mode 100644 index 00000000..2ade7b16 --- /dev/null +++ b/api/src/guest-clock.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# KVM microVM guest clock synchronization wrapper. +# Synchronizes guest system CLOCK_REALTIME directly from the host via the KVM +# virtual PTP hardware clock (/dev/ptp0) using chrony, before starting the API. +# The host clock is the root authority; no external network NTP traffic is used. +# Runtime state is isolated in a private, root-owned directory under /run. + +# Own the bootstrap shell: libkrun signals this PID, not a supervisor subshell. +# Like the previous exec path, this function exits rather than returning. +run_clock_synchronized_guest() { + set -euo pipefail + local root="${1?root argument required (empty for the guest)}" + shift + [ "$#" -gt 0 ] || { echo 'ERROR: guest command required' >&2; exit 1; } + export PATH="${PATH:-/usr/local/bin:/usr/bin:/bin}:/usr/sbin" + + local clock_dir="$root/run/codeapi-clock" + local socket_file="$clock_dir/chronyd.sock" + local chrony_pid="" guest_pid="" sync_pid="" + + # /run is a fresh guest-private tmpfs. mkdir refuses existing entries, + # including dangling symlinks; sandbox UIDs cannot replace its children. + (umask 077; mkdir "$clock_dir") || exit 1 + + stop_child() { + local pid="$1" + [ -n "$pid" ] || return 0 + kill -TERM "$pid" 2>/dev/null || true + local attempt + for attempt in {1..20}; do + kill -0 "$pid" 2>/dev/null || break + sleep 0.05 + done + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + } + + cleanup() { + trap - EXIT INT TERM + stop_child "$sync_pid" + stop_child "$guest_pid" + stop_child "$chrony_pid" + rm -rf -- "$clock_dir" + } + trap cleanup EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + chronyd -d -u root -f "$root/sandbox_api/config/guest-chrony.conf" & + chrony_pid=$! + + # Allow socket creation before chronyc connects; refuse a dead daemon. + local attempt + for attempt in {1..100}; do + kill -0 "$chrony_pid" 2>/dev/null || { + echo 'ERROR: guest clock daemon exited during startup' >&2 + exit 1 + } + [ ! -e "$socket_file" ] || break + sleep 0.05 + done + [ -e "$socket_file" ] || { + echo 'ERROR: guest clock control socket did not become ready' >&2 + exit 1 + } + + # Bound both clock acquisition and a stuck control client. Backgrounding + # the wait lets INT/TERM interrupt startup and clean up the whole group. + timeout -k 1 15 chronyc -h "$socket_file" waitsync 100 0.05 0 0.1 & + sync_pid=$! + local status=0 + wait "$sync_pid" || status=$? + sync_pid="" + if [ "$status" -ne 0 ] || ! kill -0 "$chrony_pid" 2>/dev/null; then + echo 'ERROR: guest clock synchronization failed' >&2 + exit 1 + fi + + "$@" <&0 & + guest_pid=$! + local finished="" + status=0 + wait -n -p finished "$chrony_pid" "$guest_pid" 2>/dev/null || status=$? + if [ "${finished:-}" = "$chrony_pid" ]; then + chrony_pid="" + echo 'ERROR: guest clock daemon exited; stopping the guest command' >&2 + exit 1 + fi + if [ "${finished:-}" = "$guest_pid" ]; then + guest_pid="" + exit "$status" + fi + + # Bash can cache a very short-lived child's exit before wait -n starts. + # Read its status once without swallowing failures or spinning on ECHILD. + kill -0 "$chrony_pid" 2>/dev/null || exit 1 + status=0 + wait "$guest_pid" || status=$? + guest_pid="" + exit "$status" +} + diff --git a/api/src/guest-dns.sh b/api/src/guest-dns.sh index d8371444..1a24cf91 100644 --- a/api/src/guest-dns.sh +++ b/api/src/guest-dns.sh @@ -41,7 +41,8 @@ run_guest_command() { # DNS separate from /tmp, which the normal API entrypoint mounts later. mount -t tmpfs -o size=1m,mode=0755 tmpfs "$root/run" || return 1 configure_guest_dns "$root" || return 1 - exec -- "$@" + source "$(dirname "${BASH_SOURCE[0]}")/guest-clock.sh" + run_clock_synchronized_guest "$root" "$@" } if [ "${BASH_SOURCE[0]}" = "$0" ]; then diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 824039ec..05a47fb3 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -130,6 +130,7 @@ RUN apt-install \ bsdextrautils \ locales \ procps \ + chrony \ iptables \ bc \ jq \ @@ -170,6 +171,7 @@ COPY --from=sandbox-builder /app/.build ./.build COPY api/config ./config COPY api/src/entrypoint.sh ./entrypoint.sh COPY api/src/guest-dns.sh ./guest-dns.sh +COPY api/src/guest-clock.sh ./guest-clock.sh RUN chmod +x ./entrypoint.sh RUN mkdir -p /pkgs /tmp/sandbox diff --git a/launcher/Dockerfile b/launcher/Dockerfile index 95a4d48d..c57c71ab 100644 --- a/launcher/Dockerfile +++ b/launcher/Dockerfile @@ -60,6 +60,7 @@ RUN apt-get update && \ bsdextrautils \ locales \ procps \ + chrony \ iptables \ socat \ libnl-route-3-200 \ @@ -110,6 +111,7 @@ RUN rm -f /usr/bin/nsenter /usr/bin/unshare /usr/bin/chroot /usr/sbin/chroot \ COPY api/src/entrypoint.sh ./entrypoint.sh COPY api/src/guest-dns.sh ./guest-dns.sh +COPY api/src/guest-clock.sh ./guest-clock.sh RUN chmod +x ./entrypoint.sh # ============================================================================ diff --git a/tests/guest_clock_lifecycle.sh b/tests/guest_clock_lifecycle.sh new file mode 100644 index 00000000..11f03188 --- /dev/null +++ b/tests/guest_clock_lifecycle.sh @@ -0,0 +1,419 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Keep every fixture in one bounded, private session, including GNU timeout's +# separate process group. Clean descendants even when an assertion or TERM +# interrupts the suite before the supervisor can perform its own cleanup. +if [[ "${CLOCK_TEST_SESSION:-}" != 1 ]]; then + exec python3 - "$0" "$@" <<'PY' +import os +import signal +import subprocess +import sys +import time + +child = subprocess.Popen( + ["bash", *sys.argv[1:]], + env={**os.environ, "CLOCK_TEST_SESSION": "1"}, + start_new_session=True, +) + +def stop_session(sig): + pids = subprocess.check_output(["ps", "-axo", "pid="], text=True).split() + for value in pids: + pid = int(value) + try: + if os.getsid(pid) == child.pid: + os.kill(pid, sig) + except (ProcessLookupError, PermissionError): + pass + +def interrupted(sig, _frame): + raise SystemExit(128 + sig) + +signal.signal(signal.SIGTERM, interrupted) +signal.signal(signal.SIGINT, interrupted) +try: + try: + status = child.wait(timeout=40) + except subprocess.TimeoutExpired: + print("clock lifecycle suite exceeded its deadline", file=sys.stderr) + status = 124 +finally: + stop_session(signal.SIGTERM) + time.sleep(0.1) + stop_session(signal.SIGKILL) + child.wait() +sys.exit(status) +PY +fi + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TEST_DIR="$(mktemp -d)" + +cleanup() { + # The outer session owner handles process cleanup even on suite failure. + chmod -R u+w "$TEST_DIR" 2>/dev/null || true + rm -rf "$TEST_DIR" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +create_guest_root() { + local dir="$1" + mkdir -p "$dir/run" "$dir/sandbox_api/config" + cp "$ROOT/api/config/guest-chrony.conf" "$dir/sandbox_api/config/guest-chrony.conf" +} + +# Base stub bin directory for simulating chronyd and chronyc +mkdir -p "$TEST_DIR/bin" + +cat > "$TEST_DIR/bin/chronyd" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${TEST_CHRONYD_CRASH_EARLY:-false}" == "true" ]]; then + echo "chronyd early crash simulation" >&2 + exit 1 +fi + +cfg="" +while [[ $# -gt 0 ]]; do + if [[ "$1" == "-f" ]]; then + cfg="$2" + shift 2 + else + shift + fi +done + +if [[ -z "$cfg" || ! -f "$cfg" ]]; then + echo "chronyd: configuration file not found: $cfg" >&2 + exit 1 +fi + +root="${cfg%/sandbox_api/config/guest-chrony.conf}" +clock_dir="${root}/run/codeapi-clock" +sock="${clock_dir}/chronyd.sock" + +if [[ -n "${TEST_CHRONYD_PID_FILE:-}" ]]; then + echo "$$" > "$TEST_CHRONYD_PID_FILE" +fi + +mkdir -p "$(dirname "$sock")" +touch "$sock" + +trap 'rm -f "$sock"; exit 0' TERM INT EXIT + +while true; do + sleep 0.05 +done +STUB + +cat > "$TEST_DIR/bin/chronyc" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${TEST_SYNC_FAIL:-false}" == "true" ]]; then + echo "chronyc waitsync failure simulation" >&2 + exit 1 +fi + +if [[ -n "${TEST_SYNC_STARTED_FILE:-}" ]]; then + touch "$TEST_SYNC_STARTED_FILE" +fi + +if [[ -n "${TEST_DELAY_SYNC_FILE:-}" ]]; then + while [[ ! -e "$TEST_DELAY_SYNC_FILE" ]]; do + sleep 0.05 + done +fi + +exit 0 +STUB + +chmod +x "$TEST_DIR/bin/chronyd" "$TEST_DIR/bin/chronyc" + +# Use an exec'ing executable, not a background shell function: Bash 5.2 can +# otherwise leave a wrapper PID between the test and the supervised workload. +cat > "$TEST_DIR/bin/invoke_guest" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +root="$1" +shift +exec bash -c 'source "$1"; run_clock_synchronized_guest "$2" "${@:3}"' -- \ + "$TEST_CLOCK_SOURCE" "$root" "$@" +STUB +chmod +x "$TEST_DIR/bin/invoke_guest" +export TEST_CLOCK_SOURCE="$ROOT/api/src/guest-clock.sh" +export PATH="$TEST_DIR/bin:$PATH" + +# 1. Directory pre-seed rejection +GUEST_ROOT="$TEST_DIR/root_preseed" +create_guest_root "$GUEST_ROOT" +mkdir -p "$GUEST_ROOT/run/codeapi-clock" +API_CALLED="$TEST_DIR/api_called_preseed" +if invoke_guest "$GUEST_ROOT" touch "$API_CALLED" 2>/dev/null; then + echo 'accepted pre-existing clock directory' >&2 + exit 1 +fi +[[ ! -e "$API_CALLED" ]] + +# 2. Directory symlink rejection +GUEST_ROOT="$TEST_DIR/root_symlink" +create_guest_root "$GUEST_ROOT" +mkdir -p "$TEST_DIR/foreign_dir" +ln -s "$TEST_DIR/foreign_dir" "$GUEST_ROOT/run/codeapi-clock" +API_CALLED="$TEST_DIR/api_called_symlink" +if invoke_guest "$GUEST_ROOT" touch "$API_CALLED" 2>/dev/null; then + echo 'accepted symlinked clock directory' >&2 + exit 1 +fi +[[ ! -e "$API_CALLED" ]] +[[ -z "$(ls -A "$TEST_DIR/foreign_dir")" ]] + +# 3. Mode 0700 creation and cleanup on exit +GUEST_ROOT="$TEST_DIR/root_mode" +create_guest_root "$GUEST_ROOT" +MODE_FILE="$TEST_DIR/dir_mode" +invoke_guest "$GUEST_ROOT" bash -c 'ls -ld "$1/run/codeapi-clock" | cut -c1-10 > "$2"' -- "$GUEST_ROOT" "$MODE_FILE" +[[ "$(cat "$MODE_FILE")" == "drwx------" ]] +[[ ! -e "$GUEST_ROOT/run/codeapi-clock" ]] + +# 4. Gating: API is never started before trusted clock synchronization is ready +GUEST_ROOT="$TEST_DIR/root_gate" +create_guest_root "$GUEST_ROOT" +SYNC_TRIGGER="$TEST_DIR/sync_proceed" +API_CALLED="$TEST_DIR/api_called_gate" + +TEST_SYNC_STARTED_FILE="$TEST_DIR/sync_started" \ +TEST_DELAY_SYNC_FILE="$SYNC_TRIGGER" invoke_guest "$GUEST_ROOT" touch "$API_CALLED" & +gate_pid=$! + +# Wait until chronyd has created the socket and chronyc is waiting on the trigger +while [[ ! -e "$TEST_DIR/sync_started" ]]; do + if ! kill -0 "$gate_pid" 2>/dev/null; then + echo "supervisor died unexpectedly during gating setup" >&2 + exit 1 + fi + sleep 0.05 +done + +# chronyc is now definitely blocked on synchronization, not merely unscheduled. +[[ ! -e "$API_CALLED" ]] + +# Release synchronization and verify API now executes and completes +touch "$SYNC_TRIGGER" +wait "$gate_pid" +[[ -e "$API_CALLED" ]] +[[ ! -e "$GUEST_ROOT/run/codeapi-clock" ]] + +# 5. Startup synchronization failure prevents API from starting and cleans up +GUEST_ROOT="$TEST_DIR/root_sync_fail" +create_guest_root "$GUEST_ROOT" +API_CALLED="$TEST_DIR/api_called_sync_fail" +CHRONYD_PID_FILE="$TEST_DIR/chronyd_sync_fail.pid" +if TEST_SYNC_FAIL=true \ + TEST_CHRONYD_PID_FILE="$CHRONYD_PID_FILE" \ + invoke_guest "$GUEST_ROOT" touch "$API_CALLED" 2>/dev/null; then + echo "supervisor succeeded despite synchronization failure" >&2 + exit 1 +fi +[[ ! -e "$API_CALLED" ]] +cpid="$(cat "$CHRONYD_PID_FILE" 2>/dev/null || true)" +if [[ -n "$cpid" ]] && kill -0 "$cpid" 2>/dev/null; then + echo "chronyd was not terminated on sync failure" >&2 + exit 1 +fi +[[ ! -e "$GUEST_ROOT/run/codeapi-clock" ]] + +# 6. Chronyd early crash prevents API execution +GUEST_ROOT="$TEST_DIR/root_early_crash" +create_guest_root "$GUEST_ROOT" +API_CALLED="$TEST_DIR/api_called_early_crash" +if TEST_CHRONYD_CRASH_EARLY=true \ + invoke_guest "$GUEST_ROOT" touch "$API_CALLED" 2>/dev/null; then + echo "supervisor succeeded despite chronyd early crash" >&2 + exit 1 +fi +[[ ! -e "$API_CALLED" ]] +[[ ! -e "$GUEST_ROOT/run/codeapi-clock" ]] + +# 7. Normal API exit status propagation (0) and clean daemon shutdown +GUEST_ROOT="$TEST_DIR/root_code_0" +create_guest_root "$GUEST_ROOT" +CHRONYD_PID_FILE="$TEST_DIR/chronyd_code_0.pid" +TEST_CHRONYD_PID_FILE="$CHRONYD_PID_FILE" invoke_guest "$GUEST_ROOT" bash -c "exit 0" +cpid="$(cat "$CHRONYD_PID_FILE")" +if kill -0 "$cpid" 2>/dev/null; then + echo "chronyd was not cleaned up after normal API exit" >&2 + exit 1 +fi +[[ ! -e "$GUEST_ROOT/run/codeapi-clock" ]] + +# 8. Non-zero API exit status propagation (42) and clean daemon shutdown +GUEST_ROOT="$TEST_DIR/root_code_42" +create_guest_root "$GUEST_ROOT" +CHRONYD_PID_FILE="$TEST_DIR/chronyd_code_42.pid" +set +e +TEST_CHRONYD_PID_FILE="$CHRONYD_PID_FILE" invoke_guest "$GUEST_ROOT" bash -c "exit 42" +rc=$? +set -e +[[ "$rc" -eq 42 ]] +cpid="$(cat "$CHRONYD_PID_FILE")" +if kill -0 "$cpid" 2>/dev/null; then + echo "chronyd was not cleaned up after non-zero API exit" >&2 + exit 1 +fi +[[ ! -e "$GUEST_ROOT/run/codeapi-clock" ]] + +# 9. Clock daemon death terminates active API and returns non-zero +GUEST_ROOT="$TEST_DIR/root_daemon_death" +create_guest_root "$GUEST_ROOT" +CHRONYD_PID_FILE="$TEST_DIR/chronyd_daemon_death.pid" +API_PID_FILE="$TEST_DIR/api_daemon_death.pid" +API_READY="$TEST_DIR/api_ready_daemon_death" + +TEST_CHRONYD_PID_FILE="$CHRONYD_PID_FILE" \ +invoke_guest "$GUEST_ROOT" bash -c 'echo $$ > "$1"; touch "$2"; while true; do sleep 0.05; done' -- "$API_PID_FILE" "$API_READY" & +sup_pid=$! + +while [[ ! -e "$API_READY" ]]; do + if ! kill -0 "$sup_pid" 2>/dev/null; then + echo "supervisor died prematurely during daemon death setup" >&2 + exit 1 + fi + sleep 0.05 +done + +api_pid="$(cat "$API_PID_FILE")" +chrony_pid="$(cat "$CHRONYD_PID_FILE")" +kill -0 "$api_pid" +kill -0 "$chrony_pid" + +# Kill chronyd to simulate daemon crash +kill -9 "$chrony_pid" + +set +e +wait "$sup_pid" +sup_rc=$? +set -e + +if [[ "$sup_rc" -eq 0 ]]; then + echo "supervisor returned zero exit code after clock daemon crash" >&2 + exit 1 +fi + +# The API process must have been terminated and not left running +if kill -0 "$api_pid" 2>/dev/null; then + echo "API process was left alive after clock daemon crash" >&2 + exit 1 +fi +[[ ! -e "$GUEST_ROOT/run/codeapi-clock" ]] + +# 10. Clean signal termination (SIGTERM) to child process stops both daemon and API child +GUEST_ROOT="$TEST_DIR/root_sigterm" +create_guest_root "$GUEST_ROOT" +CHRONYD_PID_FILE="$TEST_DIR/chronyd_sigterm.pid" +API_PID_FILE="$TEST_DIR/api_sigterm.pid" +API_READY="$TEST_DIR/api_ready_sigterm" + +TEST_CHRONYD_PID_FILE="$CHRONYD_PID_FILE" \ +invoke_guest "$GUEST_ROOT" bash -c 'echo $$ > "$1"; touch "$2"; while true; do sleep 0.05; done' -- "$API_PID_FILE" "$API_READY" & +sup_pid=$! + +while [[ ! -e "$API_READY" ]]; do + if ! kill -0 "$sup_pid" 2>/dev/null; then + echo "supervisor died prematurely during sigterm setup" >&2 + exit 1 + fi + sleep 0.05 +done + +api_pid="$(cat "$API_PID_FILE")" +chrony_pid="$(cat "$CHRONYD_PID_FILE")" +kill -0 "$api_pid" +kill -0 "$chrony_pid" + +kill -TERM "$sup_pid" +set +e +wait "$sup_pid" +wait_rc=$? +set -e + +[[ "$wait_rc" -eq 143 ]] + +if kill -0 "$api_pid" 2>/dev/null; then + echo "API process was left alive after supervisor SIGTERM" >&2 + exit 1 +fi +if kill -0 "$chrony_pid" 2>/dev/null; then + echo "chronyd was left alive after supervisor SIGTERM" >&2 + exit 1 +fi +[[ ! -e "$GUEST_ROOT/run/codeapi-clock" ]] + +# 11. Bounded shutdown handles stubborn child ignoring SIGTERM (SIGKILL fallback) +GUEST_ROOT="$TEST_DIR/root_stubborn" +create_guest_root "$GUEST_ROOT" +CHRONYD_PID_FILE="$TEST_DIR/chronyd_stubborn.pid" +API_PID_FILE="$TEST_DIR/api_stubborn.pid" +API_READY="$TEST_DIR/api_ready_stubborn" + +TEST_CHRONYD_PID_FILE="$CHRONYD_PID_FILE" \ +invoke_guest "$GUEST_ROOT" bash -c 'trap "" TERM; echo $$ > "$1"; touch "$2"; while true; do sleep 0.05; done' -- "$API_PID_FILE" "$API_READY" & +sup_pid=$! + +while [[ ! -e "$API_READY" ]]; do + if ! kill -0 "$sup_pid" 2>/dev/null; then + echo "supervisor died prematurely during stubborn child setup" >&2 + exit 1 + fi + sleep 0.05 +done + +api_pid="$(cat "$API_PID_FILE")" +chrony_pid="$(cat "$CHRONYD_PID_FILE")" +kill -0 "$api_pid" +kill -0 "$chrony_pid" + +start_ts=$(date +%s) +kill -TERM "$sup_pid" +set +e +wait "$sup_pid" +wait_rc=$? +set -e +end_ts=$(date +%s) + +[[ "$wait_rc" -eq 143 ]] + +if kill -0 "$api_pid" 2>/dev/null; then + echo "stubborn API child was not killed" >&2 + exit 1 +fi +if kill -0 "$chrony_pid" 2>/dev/null; then + echo "chronyd was left alive after stubborn child termination" >&2 + exit 1 +fi +elapsed=$((end_ts - start_ts)) +if [[ "$elapsed" -gt 5 ]]; then + echo "shutdown took longer than bounded limit: ${elapsed}s" >&2 + exit 1 +fi +[[ ! -e "$GUEST_ROOT/run/codeapi-clock" ]] + +# 12. Missing command or missing config file fails cleanly without executing +GUEST_ROOT="$TEST_DIR/root_empty" +create_guest_root "$GUEST_ROOT" +if invoke_guest "$GUEST_ROOT" 2>/dev/null; then + echo "accepted missing command" >&2 + exit 1 +fi + +GUEST_ROOT="$TEST_DIR/root_noconfig" +mkdir -p "$GUEST_ROOT/run" +if invoke_guest "$GUEST_ROOT" true 2>/dev/null; then + echo "accepted missing chrony config" >&2 + exit 1 +fi + +printf 'guest clock lifecycle checks passed\n' diff --git a/tests/kvm_guest_dns.sh b/tests/kvm_guest_dns.sh index 4ea99afb..b7ffed68 100755 --- a/tests/kvm_guest_dns.sh +++ b/tests/kvm_guest_dns.sh @@ -288,6 +288,32 @@ PY # The wrapper configures DNS before a custom guest executable, independently # of the normal API entrypoint and its later /tmp mount. rm -rf "$TEST_DIR/guest/run/codeapi-resolver" +mkdir -p "$TEST_DIR/guest/sandbox_api/config" +cp "$ROOT/api/config/guest-chrony.conf" "$TEST_DIR/guest/sandbox_api/config/guest-chrony.conf" +cat > "$TEST_DIR/bin/chronyd" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +cfg="" +while [[ $# -gt 0 ]]; do + if [[ "$1" == "-f" ]]; then + cfg="$2" + shift 2 + else + shift + fi +done +root="${cfg%/sandbox_api/config/guest-chrony.conf}" +sock="${root}/run/codeapi-clock/chronyd.sock" +mkdir -p "$(dirname "$sock")" +touch "$sock" +trap 'rm -f "$sock"; exit 0' TERM INT EXIT +while true; do sleep 0.1; done +STUB +cat > "$TEST_DIR/bin/chronyc" <<'STUB' +#!/usr/bin/env bash +exit 0 +STUB +chmod +x "$TEST_DIR/bin/chronyd" "$TEST_DIR/bin/chronyc" cat > "$TEST_DIR/bin/mount" <<'STUB' #!/usr/bin/env bash set -euo pipefail