From 1486c4ec4e86f6693ddbd0b0f00ba834a8feb013 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Thu, 29 Oct 2026 07:07:44 +0100 Subject: [PATCH 01/77] test(ci): run code under test where no container engine can be reached tests/devenv/unit.sh sources and runs bin/devenv, and the guard regression suites run the guards; both accept a modified or mutated copy (DEVENV=, GUARD=, ADAPTED= or an edited file). Nothing kept that code away from the Docker daemon of the machine, so a mutant with a destructive Docker command ran against the developer's real images. scripts/ci/no-docker.sh runs a command with no route to a container engine: - macOS: sandbox-exec denies Unix socket connects, TCP beyond loopback and to loopback ports 22, 2375 and 2376, writes under $HOME, open, osascript, launchctl, ssh, Launch Services and Apple Events; ps is an ad-hoc signed copy because setuid programs cannot run inside. - Linux: sudo unshare with a network namespace that has only loopback, a tmpfs over /run, then the calling user without supplementary groups and with no_new_privs. - DOCKER_* and similar variables are removed; DOCKER_CONFIG is empty. scripts/ci/no-docker-probe.py proves from inside that the mechanism is in effect and that no engine answers on the socket paths, contexts, DOCKER_HOST, sockets in the usual directories and listening loopback ports. Otherwise the command does not run (exit 125). unit.sh and the four guard suites re-execute themselves under it before they source or run anything. integration.sh, which uses the real daemon, refuses a DEVENV that differs from bin/devenv. Inside the isolation sudo is unavailable, so the U-16 cross-user read check reports INFO and is not counted (287 instead of 288 in CI). --- scripts/ci/no-docker-probe.py | 254 ++++++++++++++++++ scripts/ci/no-docker.sh | 129 +++++++++ .../tests/guard-dockerfile-frontend.test.sh | 4 + scripts/ci/tests/guard-edge-config.test.sh | 4 + scripts/ci/tests/guard-tracked-files.test.sh | 4 + scripts/ci/tests/guard-trivyignore.test.sh | 4 + tests/devenv/integration.sh | 11 +- tests/devenv/unit.sh | 4 + 8 files changed, 412 insertions(+), 2 deletions(-) create mode 100644 scripts/ci/no-docker-probe.py create mode 100755 scripts/ci/no-docker.sh diff --git a/scripts/ci/no-docker-probe.py b/scripts/ci/no-docker-probe.py new file mode 100644 index 0000000..ca79d25 --- /dev/null +++ b/scripts/ci/no-docker-probe.py @@ -0,0 +1,254 @@ +"""Prove that this process cannot reach a container engine. + +Run by scripts/ci/no-docker.sh inside its isolation, before the command it +protects, and by `no-docker.sh --check`. Exit 0 only if + + - the isolation mechanism is in effect in this process: + macOS: a connect to a Unix socket of our own and a TCP connect to + loopback port 2375 and to a non-loopback address are refused by + the sandbox (EPERM), not merely unanswered; + Linux: no_new_privs is set, no supplementary group is held, the + network namespace has only the loopback interface, and /run is a + tmpfs without any socket; + - and no container engine answers on any endpoint we can find: the + usual socket paths of Docker Engine, Docker Desktop, rootless Docker, + Podman, Colima, OrbStack and Rancher Desktop; every endpoint of the + Docker contexts under ~/.docker; DOCKER_HOST and the DOCKER_HOST the + caller of no-docker.sh had (NO_DOCKER_ORIGINAL_HOST); every socket under /run, /var/run, /tmp, + $TMPDIR and ~/.docker; loopback ports 2375 and 2376; and every TCP + port listening on loopback or on all addresses (a socket proxy or a + published engine port). + +An endpoint is an engine if it accepts a connection and answers +`GET /_ping` (sent with the User-Agent no-docker-probe) with an API version +header. The endpoints are only contacted once the mechanism is proven, so +outside the isolation nothing is sent to an engine. Prints one line per +check. +""" + +import concurrent.futures +import errno +import glob +import json +import os +import socket +import stat +import subprocess +import sys +import tempfile + +ok = True + + +def report(passed, text): + global ok + if not passed: + ok = False + print(f"no-docker: {'ok ' if passed else 'FAIL'} {text}") + + +def denied(family, address): + """True if connecting is refused by the sandbox (EPERM/EACCES).""" + s = socket.socket(family, socket.SOCK_STREAM) + s.settimeout(2) + try: + s.connect(address) + except OSError as e: + return e.errno in (errno.EPERM, errno.EACCES) + finally: + s.close() + return False + + +def engine_answers(family, address, timeout=2): + """Return a description if an engine answers at address, else None.""" + s = socket.socket(family, socket.SOCK_STREAM) + s.settimeout(timeout) + try: + s.connect(address) + s.sendall(b"GET /_ping HTTP/1.0\r\nHost: docker\r\nUser-Agent: no-docker-probe\r\n\r\n") + data = b"" + # Stop at the end of the headers: an engine may keep the connection + # open (HTTP/1.1 keep-alive) instead of closing it after the answer. + while len(data) < 4096 and b"\r\n\r\n" not in data: + try: + chunk = s.recv(4096) + except OSError: + break + if not chunk: + break + data += chunk + except OSError: + return None + finally: + s.close() + head = data.split(b"\r\n\r\n", 1)[0].lower() + if b"api-version:" in head or b"libpod-api-version:" in head: + return head.split(b"\r\n", 1)[0].decode("ascii", "replace") + return None + + +def check_mechanism_darwin(): + d = tempfile.mkdtemp(prefix="no-docker-probe.") + path = os.path.join(d, "s") + srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + srv.bind(path) + srv.listen(1) + report(denied(socket.AF_UNIX, path), "macOS sandbox denies Unix socket connects") + finally: + srv.close() + os.unlink(path) + os.rmdir(d) + report(denied(socket.AF_INET, ("127.0.0.1", 2375)), "macOS sandbox denies loopback port 2375") + report(denied(socket.AF_INET, ("192.0.2.1", 2375)), "macOS sandbox denies non-loopback TCP") + + +def check_mechanism_linux(): + nnp = "" + groups = "" + with open("/proc/self/status") as f: + for line in f: + if line.startswith("NoNewPrivs:"): + nnp = line.split()[1] + elif line.startswith("Groups:"): + groups = line[len("Groups:"):].strip() + report(nnp == "1", f"no_new_privs is set ({nnp or 'unknown'})") + report(groups == "", f"no supplementary groups ({groups or 'none'})") + names = sorted(n for _, n in socket.if_nameindex()) + report(names == ["lo"], f"network namespace has only loopback ({' '.join(names)})") + run_tmpfs = False + with open("/proc/self/mountinfo") as f: + for line in f: + fields = line.split() + sep = fields.index("-") + if fields[4] == "/run" and fields[sep + 1] == "tmpfs": + run_tmpfs = True # the last mount on /run is the visible one + elif fields[4] == "/run": + run_tmpfs = False + report(run_tmpfs, "/run is a private tmpfs") + sockets = [p for p in walk_sockets(["/run"])] + report(not sockets, f"no socket under /run ({' '.join(sockets) or 'none'})") + + +def walk_sockets(roots, limit=20000): + seen = 0 + for root in roots: + if not root or not os.path.isdir(root): + continue + for dirpath, dirnames, filenames in os.walk(root, onerror=lambda e: None): + for name in dirnames + filenames: + seen += 1 + if seen > limit: + return + p = os.path.join(dirpath, name) + try: + if stat.S_ISSOCK(os.lstat(p).st_mode): + yield p + except OSError: + pass + + +def listening_ports(): + """(host, port) of TCP ports listening on loopback or on all addresses.""" + ports = set() + if sys.platform == "darwin": + try: + out = subprocess.run(["netstat", "-an", "-p", "tcp"], capture_output=True, text=True, timeout=10).stdout + except (OSError, subprocess.SubprocessError): + out = "" + for line in out.splitlines(): + f = line.split() + if len(f) >= 6 and f[-1] == "LISTEN": + host, _, port = f[3].rpartition(".") + if host in ("127.0.0.1", "::1", "*") and port.isdigit(): + ports.add(("::1" if host == "::1" else "127.0.0.1", int(port))) + else: + for name in ("/proc/net/tcp", "/proc/net/tcp6"): + try: + with open(name) as f: + next(f) + for line in f: + local, state = line.split()[1], line.split()[3] + if state == "0A": + ports.add(("::1" if name.endswith("6") else "127.0.0.1", int(local.rsplit(":", 1)[1], 16))) + except OSError: + pass + return ports + + +def endpoints(): + home = os.path.expanduser("~") + uid = os.getuid() + unix = { + "/var/run/docker.sock", + "/run/docker.sock", + "/run/containerd/containerd.sock", + "/run/podman/podman.sock", + f"/run/user/{uid}/docker.sock", + f"/run/user/{uid}/podman/podman.sock", + f"{home}/.docker/run/docker.sock", + f"{home}/.docker/desktop/docker.sock", + f"{home}/.colima/default/docker.sock", + f"{home}/.orbstack/run/docker.sock", + f"{home}/.rd/docker.sock", + f"{home}/Library/Containers/com.docker.docker/Data/docker.raw.sock", + f"{home}/Library/Containers/com.docker.docker/Data/docker-cli.sock", + } + tcp = {("127.0.0.1", 2375), ("127.0.0.1", 2376), ("::1", 2375), ("::1", 2376)} + tcp.update(listening_ports()) + hosts = [os.environ.get("NO_DOCKER_ORIGINAL_HOST", ""), os.environ.get("DOCKER_HOST", "")] + for meta in glob.glob(f"{home}/.docker/contexts/meta/*/meta.json"): + try: + with open(meta) as f: + hosts.append(json.load(f)["Endpoints"]["docker"]["Host"]) + except (OSError, ValueError, KeyError, TypeError): + pass + for h in hosts: + if h.startswith("unix://"): + unix.add(h[len("unix://"):]) + elif h.startswith("tcp://"): + hostport = h[len("tcp://"):].split("/", 1)[0] + host, _, port = hostport.rpartition(":") + if host and port.isdigit(): + tcp.add((host.strip("[]"), int(port))) + roots = ["/run", "/var/run", "/tmp", os.environ.get("TMPDIR", ""), f"{home}/.docker", + f"{home}/Library/Containers/com.docker.docker/Data"] + unix.update(walk_sockets(roots)) + return sorted(unix), sorted(tcp) + + +def main(): + if sys.platform == "darwin": + check_mechanism_darwin() + elif sys.platform.startswith("linux"): + check_mechanism_linux() + else: + report(False, f"no isolation mechanism for platform {sys.platform}") + if not ok: + # Not isolated: do not send anything to an engine that may answer. + return 1 + unix, tcp = endpoints() + def ask(target): + kind, addr = target + try: + if kind == "unix": + a = engine_answers(socket.AF_UNIX, addr) + else: + a = engine_answers(socket.AF_INET6 if ":" in addr[0] else socket.AF_INET, addr, 1) + except OSError: + a = None + if not a: + return None + return f"unix://{addr} ({a})" if kind == "unix" else f"tcp://{addr[0]}:{addr[1]} ({a})" + + targets = [("unix", p) for p in unix] + [("tcp", t) for t in tcp] + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: + found = [r for r in pool.map(ask, targets) if r] + report(not found, f"no container engine answers on {len(unix)} socket and {len(tcp)} TCP endpoints" + + (": " + ", ".join(found) if found else "")) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh new file mode 100755 index 0000000..d18cef3 --- /dev/null +++ b/scripts/ci/no-docker.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# Run a command where no container engine can be reached, so a test suite, +# and any modified or mutated copy of the code it runs, cannot touch the +# Docker daemon of the machine (its images, containers, volumes, networks +# or build cache) by the routes below: docker found on PATH or by an +# absolute path, in a nested shell or a command substitution, with -H, +# --context or an inherited DOCKER_HOST, curl --unix-socket or a raw socket. +# +# Usage: no-docker.sh COMMAND [ARG...] run COMMAND isolated; its status +# no-docker.sh --check exit 0 only if this process is +# already isolated +# +# Isolation: +# macOS sandbox-exec: every connect to a Unix socket is denied, as is TCP +# to anything but loopback and to loopback ports 22, 2375 and 2376 +# (ssh and the Docker API ports); nothing under $HOME can be +# written (Docker Desktop keeps its disk image there); open, +# osascript, launchctl and ssh cannot run and Launch Services and +# Apple Events cannot be reached (the known ways to have a program +# started outside the sandbox). The profile filters these routes; +# it is not a complete jail. Setuid programs cannot run in the sandbox, so +# ps is an unprivileged, ad-hoc signed copy. Loopback TCP to other +# ports stays open for the tests; the probe asks every listening +# port whether it is an engine (a socket proxy, for example). +# Linux sudo unshare: a network namespace with only loopback and a mount +# namespace with an empty tmpfs over /run (and /var/run), then the +# calling user without supplementary groups (no docker group) and +# with no_new_privs, so sudo cannot regain root to undo it. +# In both, DOCKER_* and similar variables are removed and DOCKER_CONFIG is +# an empty directory. Before COMMAND runs, no-docker-probe.py proves from +# inside that the mechanism is in effect and that no engine answers on any +# endpoint it can find. If isolation cannot be applied or proven, COMMAND +# does not run and the exit status is 125; so is the status if the +# isolation tool returns without the command having started inside it. + +set -u + +here=$(cd -P "$(dirname "$0")" && pwd -P) +probe="$here/no-docker-probe.py" +EX_REFUSED=125 + +refuse() { + printf 'no-docker: refused: %s; the command was not run\n' "$1" >&2 + exit "$EX_REFUSED" +} + +if [ "${1-}" = --check ]; then + exec python3 -I "$probe" +fi + +if [ "${1-}" = --inner ]; then + # Inside the isolation: prove it, then run the command. + shift + python3 -I "$probe" >&2 || refuse "the isolation could not be proven" + : >"$NO_DOCKER_STARTED" || refuse "cannot record the start" + unset NO_DOCKER_STARTED + exec "$@" +fi + +[ $# -gt 0 ] || refuse "no command given" +if [ -n "${NO_DOCKER_ACTIVE-}" ]; then + # Already isolated (a suite re-executing itself): never nest. + python3 -I "$probe" >/dev/null || refuse "inside no-docker.sh, but the isolation check fails" + exec "$@" +fi + +work=$(mktemp -d "${TMPDIR:-/tmp}/no-docker.XXXXXX") || refuse "no temporary directory" +work=$(cd -P "$work" && pwd -P) +trap 'rm -rf "$work"' EXIT +mkdir "$work/docker-config" "$work/bin" || refuse "cannot prepare $work" + +# The environment of the command: the caller's, without the variables that +# select a container engine or its configuration. +envargs=() +while IFS= read -r -d '' kv; do + case ${kv%%=*} in + DOCKER_* | CONTAINER_HOST | CONTAINER_CONNECTION | CONTAINERD_* | BUILDKIT_HOST | COMPOSE_* | PODMAN_* | NO_DOCKER_*) ;; + *) envargs+=("$kv") ;; + esac +done < <(env -0) +envargs+=("DOCKER_CONFIG=$work/docker-config" "NO_DOCKER_ACTIVE=1" "NO_DOCKER_ORIGINAL_HOST=${DOCKER_HOST-}" + "NO_DOCKER_STARTED=$work/started") + +case $(uname -s) in + Darwin) + command -v sandbox-exec >/dev/null || refuse "sandbox-exec not found" + # A copy of ps loses its setuid bit; it needs a signature of its own. + { cp /bin/ps "$work/bin/ps" && codesign -f -s - "$work/bin/ps" 2>/dev/null; } || + refuse "cannot prepare an unprivileged ps" + for i in "${!envargs[@]}"; do + case ${envargs[$i]} in PATH=*) envargs[i]="PATH=$work/bin:${envargs[$i]#PATH=}" ;; esac + done + home=$(cd -P "$HOME" && pwd -P) + profile="(version 1) +(allow default) +(deny network-outbound (remote unix-socket)) +(deny network-outbound (remote ip)) +(allow network-outbound (remote ip \"localhost:*\")) +(deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")) +(deny file-write* (subpath \"$home\")) +(deny process-exec (literal \"/usr/bin/open\") (literal \"/usr/bin/osascript\") (literal \"/bin/launchctl\") (literal \"/usr/bin/ssh\")) +(deny mach-lookup (global-name \"com.apple.coreservices.launchservicesd\") (global-name \"com.apple.coreservices.appleevents\") (global-name-regex #\"^com\\.apple\\.lsd\\.\"))" + sandbox-exec -p "$profile" /usr/bin/env -i "${envargs[@]}" "$BASH" "$here/no-docker.sh" --inner "$@" + ;; + Linux) + for tool in unshare setpriv ip; do + command -v "$tool" >/dev/null || refuse "$tool not found" + done + sudo -n true 2>/dev/null || refuse "creating the namespaces needs passwordless sudo" + # shellcheck disable=SC2016 # expanded by the inner shell + sudo -n unshare --net --mount --propagation private -- /bin/sh -c ' + ip link set lo up || exit 125 + mount -t tmpfs -o mode=0755,nosuid,nodev,noexec no-docker /run || exit 125 + if [ -d /var/run ] && [ ! -L /var/run ]; then + mount -t tmpfs -o mode=0755,nosuid,nodev,noexec no-docker /var/run || exit 125 + fi + uid=$1 gid=$2 + shift 2 + exec setpriv --reuid "$uid" --regid "$gid" --clear-groups --no-new-privs -- /usr/bin/env -i "$@" + ' no-docker "$(id -u)" "$(id -g)" "${envargs[@]}" "$BASH" "$here/no-docker.sh" --inner "$@" + ;; + *) + refuse "no isolation mechanism for $(uname -s)" + ;; +esac +rc=$? +[ -e "$work/started" ] || refuse "the isolation tool returned $rc without starting the command" +exit "$rc" diff --git a/scripts/ci/tests/guard-dockerfile-frontend.test.sh b/scripts/ci/tests/guard-dockerfile-frontend.test.sh index 623db6a..0deb7d2 100755 --- a/scripts/ci/tests/guard-dockerfile-frontend.test.sh +++ b/scripts/ci/tests/guard-dockerfile-frontend.test.sh @@ -10,6 +10,10 @@ set -u here=$(cd "$(dirname "$0")" && pwd) +# The suite runs the guard, or a modified or mutated copy of it (GUARD=), +# so it runs only where no container engine can be reached. +"$here/../no-docker.sh" --check >/dev/null 2>&1 || + exec "$here/../no-docker.sh" "$BASH" "$here/guard-dockerfile-frontend.test.sh" "$@" guard=${GUARD:-"$here/../guard-dockerfile-frontend.sh"} real=$(git -C "$here" rev-parse --show-toplevel) || exit 2 work=$(mktemp -d "${TMPDIR:-/tmp}/guard-frontend-tests.XXXXXX") || exit 2 diff --git a/scripts/ci/tests/guard-edge-config.test.sh b/scripts/ci/tests/guard-edge-config.test.sh index 5963a6f..1cfe4b1 100755 --- a/scripts/ci/tests/guard-edge-config.test.sh +++ b/scripts/ci/tests/guard-edge-config.test.sh @@ -14,6 +14,10 @@ set -u here=$(cd "$(dirname "$0")" && pwd) +# The suite runs the guard, or a modified or mutated copy of it (GUARD=), +# so it runs only where no container engine can be reached. +"$here/../no-docker.sh" --check >/dev/null 2>&1 || + exec "$here/../no-docker.sh" "$BASH" "$here/guard-edge-config.test.sh" "$@" guard=${GUARD:-"$here/../guard-edge-config.sh"} adapted=${ADAPTED:-"$here/../edge-adapted-config.jq"} real=$(git -C "$here" rev-parse --show-toplevel) || exit 2 diff --git a/scripts/ci/tests/guard-tracked-files.test.sh b/scripts/ci/tests/guard-tracked-files.test.sh index 681c3ad..40d0f34 100755 --- a/scripts/ci/tests/guard-tracked-files.test.sh +++ b/scripts/ci/tests/guard-tracked-files.test.sh @@ -10,6 +10,10 @@ set -u here=$(cd "$(dirname "$0")" && pwd) +# The suite runs the guard, or a modified or mutated copy of it (GUARD=), +# so it runs only where no container engine can be reached. +"$here/../no-docker.sh" --check >/dev/null 2>&1 || + exec "$here/../no-docker.sh" "$BASH" "$here/guard-tracked-files.test.sh" "$@" guard=${GUARD:-"$here/../guard-tracked-files.sh"} work=$(mktemp -d "${TMPDIR:-/tmp}/guard-tests.XXXXXX") || exit 2 trap 'rm -rf "$work"' EXIT diff --git a/scripts/ci/tests/guard-trivyignore.test.sh b/scripts/ci/tests/guard-trivyignore.test.sh index a34117c..2e117da 100755 --- a/scripts/ci/tests/guard-trivyignore.test.sh +++ b/scripts/ci/tests/guard-trivyignore.test.sh @@ -10,6 +10,10 @@ set -u here=$(cd "$(dirname "$0")" && pwd) +# The suite runs the guard, or a modified or mutated copy of it (GUARD=), +# so it runs only where no container engine can be reached. +"$here/../no-docker.sh" --check >/dev/null 2>&1 || + exec "$here/../no-docker.sh" "$BASH" "$here/guard-trivyignore.test.sh" "$@" guard=${GUARD:-"$here/../guard-trivyignore.sh"} real=$(git -C "$here" rev-parse --show-toplevel) || exit 2 work=$(mktemp -d "${TMPDIR:-/tmp}/guard-trivyignore-tests.XXXXXX") || exit 2 diff --git a/tests/devenv/integration.sh b/tests/devenv/integration.sh index f67aec8..556e9ab 100755 --- a/tests/devenv/integration.sh +++ b/tests/devenv/integration.sh @@ -8,7 +8,10 @@ # # Usage: tests/devenv/integration.sh # DEVENV_TEST_PORT_BASE first port of the 20-port test range (default 8080) -# DEVENV bin/devenv copy to test (default: this checkout's) +# DEVENV accepted only if identical to this checkout's +# bin/devenv: this suite uses the real Docker daemon, +# so a modified or mutated copy belongs in +# tests/devenv/unit.sh, which runs without one # DEVENV_TEST_SCENARIOS space-separated subset, e.g. "pa16 pa21" (default: # all); the summary names the scenarios that ran # Needs: docker with Compose v2, python3 and curl (test tools only). @@ -23,6 +26,10 @@ set -u here=$(cd -P "$(dirname "$0")" && pwd -P) repo=$(cd -P "$here/../.." && pwd -P) +if [ -n "${DEVENV-}" ] && ! cmp -s "$DEVENV" "$repo/bin/devenv"; then + echo "REFUSED: DEVENV=$DEVENV differs from $repo/bin/devenv; this suite uses the real Docker daemon, run tests/devenv/unit.sh for a modified copy" >&2 + exit 2 +fi BASE=${DEVENV_TEST_PORT_BASE:-8080} LO=$BASE HI=$((BASE + 19)) @@ -97,7 +104,7 @@ new_checkout() { # name -> sets CO CO=$work/$1 mkdir -p "$CO/bin" cp "$repo/Makefile" "$repo/compose.fixture.yaml" "$CO/" - cp "${DEVENV:-$repo/bin/devenv}" "$CO/bin/devenv" + cp "$repo/bin/devenv" "$CO/bin/devenv" printf '%s\n' "$CO" >>"$work/checkouts" } diff --git a/tests/devenv/unit.sh b/tests/devenv/unit.sh index f8f9a24..a8041d1 100755 --- a/tests/devenv/unit.sh +++ b/tests/devenv/unit.sh @@ -19,6 +19,10 @@ set -u here=$(cd -P "$(dirname "$0")" && pwd -P) +# The suite sources and runs bin/devenv, or a modified or mutated copy of +# it, so it runs only where no container engine can be reached. +"$here/../../scripts/ci/no-docker.sh" --check >/dev/null 2>&1 || + exec "$here/../../scripts/ci/no-docker.sh" "$BASH" "$here/unit.sh" "$@" # DEVENV= runs the suite against another copy (mutation checks). devenv_script=${DEVENV:-"$here/../../bin/devenv"} # shellcheck source=../../bin/devenv From 1fe73369b816d4432b0648bd81d9ec5cfd8826b7 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Thu, 29 Oct 2026 07:07:44 +0100 Subject: [PATCH 02/77] test(ci): check the Docker isolation with a mutated bin/devenv scripts/ci/tests/no-docker.test.sh inserts a read-only payload into a copy of bin/devenv, passed as DEVENV and edited into the file. The payload reaches Docker in 22 ways: inherited DOCKER_HOST, PATH lookup, absolute paths, command, env, --context, nested bash and sh, command substitution, backticks, a variable, a quoted name, exec, xargs, a background subshell, sudo, curl --unix-socket, a raw Python socket and the real socket paths. Its target is a canary engine that answers like Docker and records every request. - T-00: the read-only lint rejects non-read-only payloads; the payload itself is pinned by sha256, as the macOS cases run on the host. - T-01/T-02: --check fails outside the isolation; a claimed but unproven isolation is refused. - T-03/T-04: on macOS the mutant runs to its end with no request reaching the canary and the real socket paths refused; on Linux the harness refuses because the canary is a reachable engine. - T-05: the same mutant under unit.sh without the re-execution reaches the canary, behind an outer guard that leaves only the canary reachable. Run with HARNESS= the suite fails T-03 and T-04. CI runs it in Repository guard; ShellCheck covers the new scripts. --- .github/workflows/ci.yml | 4 + scripts/ci/tests/no-docker.test.sh | 357 +++++++++++++++++++++++++++++ 2 files changed, 361 insertions(+) create mode 100755 scripts/ci/tests/no-docker.test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1120940..c5479da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,9 @@ jobs: - name: Run edge configuration guard regression tests run: scripts/ci/tests/guard-edge-config.test.sh + - name: Run test isolation regression tests + run: scripts/ci/tests/no-docker.test.sh + - name: ShellCheck run: >- docker run --rm --network none -v "$PWD:/mnt:ro" -w /mnt @@ -57,6 +60,7 @@ jobs: scripts/ci/guard-dockerfile-frontend.sh scripts/ci/tests/guard-dockerfile-frontend.test.sh scripts/ci/guard-trivyignore.sh scripts/ci/tests/guard-trivyignore.test.sh scripts/ci/guard-edge-config.sh scripts/ci/tests/guard-edge-config.test.sh + scripts/ci/no-docker.sh scripts/ci/tests/no-docker.test.sh - name: actionlint run: >- diff --git a/scripts/ci/tests/no-docker.test.sh b/scripts/ci/tests/no-docker.test.sh new file mode 100755 index 0000000..8c6ec7f --- /dev/null +++ b/scripts/ci/tests/no-docker.test.sh @@ -0,0 +1,357 @@ +#!/usr/bin/env bash +# +# Regression tests for scripts/ci/no-docker.sh and its use by +# tests/devenv/unit.sh: a mutated bin/devenv that runs Docker commands must +# be refused or neutralized, whether it is passed as DEVENV or edited in +# place, and whichever way it reaches the Docker CLI or the API. +# +# Nothing here can change anything in the Docker daemon of this machine: +# - the mutant's payload only reads (`docker version`, GET /version and +# /_ping); it never removes, prunes or creates anything. A pre-flight +# check refuses to run any case unless every Docker command of the +# payload is `version` and every API request a GET of /version or /_ping; +# - its target is a canary engine of this suite: a Unix socket that +# answers like Docker and records every request; two cases name the real +# socket paths and must be refused at the socket; +# - the harness without the containment (T-05) runs, with its canary, +# behind an outer guard that leaves only the canary reachable: on macOS +# a sandbox-exec profile denying every other Unix socket, TCP to +# 22/2375/2376 and beyond loopback, writes under $HOME and starting +# programs outside the sandbox; on Linux no-docker.sh itself, whose +# probe runs before the canary is started inside it; +# - the harness under test (T-03, T-04) runs on Linux behind the same +# guard, where it must refuse because the canary is a reachable engine. +# On macOS it runs on the host itself, as a developer would run it: +# macOS applies a second, different sandbox profile inside a sandbox +# only if it is identical, so the harness's own isolation cannot be +# tested behind the guard. Should that isolation fail, the read-only +# payload would reach the real daemon, which the cases detect. +# This suite itself runs no docker command. +# +# Usage: no-docker.test.sh +# HARNESS= checks another copy of tests/devenv/unit.sh instead of the +# current one, with every case behind the outer guard (it is unreviewed +# code); a harness without the containment must fail T-03 and T-04. On +# macOS a harness with it fails them too, as it cannot nest its sandbox. + +# shellcheck disable=SC2016 +# Single-quoted $... in sed and grep patterns and in the mutant stay literal. +# shellcheck disable=SC2319 +# "; result NAME $?" records the status of that condition on purpose. + +set -u + +here=$(cd -P "$(dirname "$0")" && pwd -P) +root=$(cd -P "$here/../../.." && pwd -P) +work=$(mktemp -d "${TMPDIR:-/tmp}/no-docker-tests.XXXXXX") || exit 2 +work=$(cd -P "$work" && pwd -P) +[ -n "$work" ] && [ -d "$work" ] || exit 2 +home=$(cd -P "$HOME" && pwd -P) +[ -n "$home" ] && [ -d "$home" ] || exit 2 +nonce=$$-$(date +%s) +canary=$work/canary.sock +home_marker=$home/.flowcrm-no-docker-test-$nonce +trap 'rm -f "$home_marker"; rm -rf "$work"' EXIT +pass=0 +fail=0 + +result() { # name ok(0|1) [detail] + if [ "$2" -eq 0 ]; then + pass=$((pass + 1)) + printf 'PASS %s\n' "$1" + else + fail=$((fail + 1)) + printf 'FAIL %s%s\n' "$1" "${3:+ $3}" + fi +} + +# --- the canary engine -------------------------------------------------------- +cat >"$work/canary.py" <<'PY' +import json, os, socketserver, sys +path, log = sys.argv[1], sys.argv[2] +class H(socketserver.StreamRequestHandler): + def handle(self): + while True: + line = self.rfile.readline() + if not line: + return + headers = {} + while True: + h = self.rfile.readline() + if h in (b"\r\n", b"\n", b""): + break + k, _, v = h.decode("latin-1").partition(":") + headers[k.strip().lower()] = v.strip() + if headers.get("user-agent") != "no-docker-probe": + with open(log, "a") as f: + f.write(line.decode("latin-1").strip() + " ua=" + headers.get("user-agent", "-") + "\n") + target = line.split()[1].decode("latin-1") if len(line.split()) > 1 else "/" + if target.endswith("/_ping"): + body = b"OK" + elif target.endswith("/version"): + body = json.dumps({"Version": "canary-engine", "ApiVersion": "1.45", "MinAPIVersion": "1.24", + "Os": "linux", "Arch": "amd64"}).encode() + else: + body = json.dumps({"message": "canary-engine"}).encode() + self.wfile.write(b"HTTP/1.1 200 OK\r\nApi-Version: 1.45\r\nOstype: linux\r\n" + b"Content-Type: application/json\r\nContent-Length: %d\r\n\r\n" % len(body)) + if not line.startswith(b"HEAD"): + self.wfile.write(body) + self.wfile.flush() +class S(socketserver.ThreadingMixIn, socketserver.UnixStreamServer): + daemon_threads = True +S(path, H).serve_forever() +PY + +# drive.sh : runs inside the outer guard. +# Starts the canary, records that the harness was started, runs it with +# DOCKER_HOST pointing at the canary and stops the canary again. +cat >"$work/drive.sh" <<'DRIVE' +dir=$1 canary=$2 log=$3 +shift 3 +rm -f "$canary" +python3 -I "$CANARY_PY" "$canary" "$log" & +cpid=$! +for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do [ -S "$canary" ] && break; sleep 0.2; done +[ -S "$canary" ] || { echo "HARNESS ERROR: the canary engine did not start"; kill "$cpid"; exit 2; } +: >"$dir/started" +DOCKER_HOST=unix://$canary "$@" +rc=$? +sleep 0.5 # let the canary finish writing +kill "$cpid" 2>/dev/null +wait "$cpid" 2>/dev/null +exit "$rc" +DRIVE + +export CANARY_PY=$work/canary.py + +# A Docker configuration whose context "canary" points at the canary. +ctx=$(printf 'canary' | { if command -v sha256sum >/dev/null 2>&1; then sha256sum; else shasum -a 256; fi; } | cut -c1-64) +mkdir -p "$work/ctxcfg/contexts/meta/$ctx" +printf '{"Name":"canary","Metadata":{},"Endpoints":{"docker":{"Host":"unix://%s","SkipTLSVerify":false}}}\n' \ + "$canary" >"$work/ctxcfg/contexts/meta/$ctx/meta.json" + +# --- the mutant --------------------------------------------------------------- +# Inserted after the first line of bin/devenv, so it runs as soon as the file +# is sourced or executed. Each case appends " rc= out=" +# to the log; the mutant then exits 97. +cat >"$work/payload" <<'MUT' +qa_log=@LOG@ qa_canary=@CANARY@ qa_cfg=@CFG@ qa_marker=@MARKER@ +qa() { # case command... + qa_n=$1 + shift + qa_out=$("$@" 2>&1) + printf '%s rc=%s out=%s\n' "$qa_n" "$?" "$(printf '%s' "$qa_out" | tr '\n' ' ' | cut -c1-200)" >>"$qa_log" +} +qa_fmt='{{.Server.Version}}' +qa inherited-env docker version --format "$qa_fmt" +qa path-lookup docker -H "unix://$qa_canary" version --format "$qa_fmt" +for qa_p in /usr/local/bin/docker /usr/bin/docker /opt/homebrew/bin/docker /Applications/Docker.app/Contents/Resources/bin/docker; do + [ ! -x "$qa_p" ] || qa "absolute-path:$qa_p" "$qa_p" -H "unix://$qa_canary" version --format "$qa_fmt" +done +qa builtin-command command docker -H "unix://$qa_canary" version --format "$qa_fmt" +qa env-wrapper env docker -H "unix://$qa_canary" version --format "$qa_fmt" +qa env-docker-host env DOCKER_HOST="unix://$qa_canary" docker version --format "$qa_fmt" +qa context env DOCKER_CONFIG="$qa_cfg" docker --context canary version --format "$qa_fmt" +qa nested-bash bash -c 'docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" +qa nested-sh sh -c 'docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" +qa_x=$(docker -H "unix://$qa_canary" version --format "$qa_fmt" 2>&1) +printf 'command-substitution rc=%s out=%s\n' "$?" "$(printf '%s' "$qa_x" | tr '\n' ' ')" >>"$qa_log" +qa_x=`docker -H "unix://$qa_canary" version --format "$qa_fmt" 2>&1` +printf 'backticks rc=%s out=%s\n' "$?" "$(printf '%s' "$qa_x" | tr '\n' ' ')" >>"$qa_log" +qa_dk=docker +qa variable "$qa_dk" -H "unix://$qa_canary" version --format "$qa_fmt" +qa quoted "docker" -H "unix://$qa_canary" version --format "$qa_fmt" +qa exec-subshell sh -c 'exec docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" +qa xargs sh -c 'echo version | xargs docker -H "unix://$1"' _ "$qa_canary" +(docker -H "unix://$qa_canary" version --format "$qa_fmt" >"$qa_log.bg" 2>&1; echo "background rc=$? out=$(tr '\n' ' ' <"$qa_log.bg")" >>"$qa_log") & +wait +qa sudo sudo -n docker -H "unix://$qa_canary" version --format "$qa_fmt" +qa curl-unix-socket curl -sS --max-time 5 --unix-socket "$qa_canary" http://docker/version +qa python-raw-socket python3 -c 'import socket,sys; s=socket.socket(socket.AF_UNIX); s.connect(sys.argv[1]); s.sendall(b"GET /version HTTP/1.0\r\n\r\n"); print(s.recv(4096).decode())' "$qa_canary" +qa real-socket-cli docker -H unix:///var/run/docker.sock version --format "$qa_fmt" +qa real-socket-curl curl -sS --max-time 5 --unix-socket /var/run/docker.sock http://docker/_ping +qa home-write touch "$qa_marker" +exit 97 +MUT + +# mutant : print bin/devenv with the payload after line 1. +mutant() { + sed -n '1p' "$1" + sed -e "s|@LOG@|'$2'|" -e "s|@CANARY@|'$canary'|" -e "s|@CFG@|'$work/ctxcfg'|" -e "s|@MARKER@|'$home_marker'|" "$work/payload" + sed '1d' "$1" +} + +# tree : a copy of the files the harness needs, with +# as tests/devenv/unit.sh. +tree() { + mkdir -p "$1/bin" "$1/tests/devenv" "$1/scripts/ci" + cp "$root/bin/devenv" "$1/bin/devenv" + cp "$2" "$1/tests/devenv/unit.sh" + cp "$root/tests/devenv/tally.sh" "$root/tests/devenv/port-holder.py" "$1/tests/devenv/" + cp "$root/scripts/ci/no-docker.sh" "$root/scripts/ci/no-docker-probe.py" "$1/scripts/ci/" + chmod +x "$1/bin/devenv" "$1/tests/devenv/unit.sh" "$1/scripts/ci/no-docker.sh" + mkdir "$1/sub" +} + +# guarded : run the command behind the outer guard +# (bare: on macOS without it, see the header). +guarded() { + local mode=$1 + shift + case $(uname -s)-$mode in + Darwin-bare) "$BASH" "$work/drive.sh" "$@" ;; + Darwin-guard) + sandbox-exec -p "(version 1) +(allow default) +(deny network-outbound (remote unix-socket)) +(allow network-outbound (remote unix-socket (path-literal \"$canary\"))) +(deny network-outbound (remote ip)) +(allow network-outbound (remote ip \"localhost:*\")) +(deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")) +(deny file-write* (subpath \"$home\")) +(deny process-exec (literal \"/usr/bin/open\") (literal \"/usr/bin/osascript\") (literal \"/bin/launchctl\") (literal \"/usr/bin/ssh\")) +(deny mach-lookup (global-name \"com.apple.coreservices.launchservicesd\") (global-name \"com.apple.coreservices.appleevents\") (global-name-regex #\"^com\\.apple\\.lsd\\.\"))" \ + "$BASH" "$work/drive.sh" "$@" + ;; + *) "$root/scripts/ci/no-docker.sh" "$BASH" "$work/drive.sh" "$@" ;; + esac +} + +# run_case : run the harness against the +# mutant; sets RC, LOG, OUT, STARTED (the harness was started inside the +# guard) and REQ (requests the canary engine received). +run_case() { + local dir=$work/$1 + tree "$dir" "$2" + LOG=$dir/mutant.log + OUT=$dir/out + : >"$LOG" + : >"$dir/canary.log" + mutant "$root/bin/devenv" "$LOG" >"$dir/mutant" + rm -f "$home_marker" + if [ "$3" = inplace ]; then + cp "$dir/mutant" "$dir/bin/devenv" + guarded "$4" "$dir/sub" "$canary" "$dir/canary.log" "$BASH" "$dir/tests/devenv/unit.sh" >"$OUT" 2>&1 + else + guarded "$4" "$dir/sub" "$canary" "$dir/canary.log" /usr/bin/env DEVENV="$dir/mutant" "$BASH" "$dir/tests/devenv/unit.sh" >"$OUT" 2>&1 + fi + RC=$? + STARTED=1 + [ -e "$dir/sub/started" ] || STARTED=0 + REQ=$(grep -c . "$dir/canary.log") +} + +# The harness without the containment: the lines that re-execute it under +# no-docker.sh removed. +sed '/^# The suite sources and runs bin\/devenv/,/scripts\/ci\/no-docker.sh" "\$BASH"/d' \ + "$root/tests/devenv/unit.sh" >"$work/unit-uncontained.sh" +if cmp -s "$work/unit-uncontained.sh" "$root/tests/devenv/unit.sh" || grep -q no-docker "$work/unit-uncontained.sh" || + ! grep -q '^\. "\$devenv_script"$' "$work/unit-uncontained.sh"; then + echo "HARNESS ERROR: could not remove the containment from tests/devenv/unit.sh" >&2 + exit 2 +fi +harness=${HARNESS:-"$root/tests/devenv/unit.sh"} +# Another harness is unreviewed code: run every case behind the guard. +t0304=bare +[ -z "${HARNESS-}" ] || t0304=guard + +# The payload is exactly the reviewed text: on macOS the cases T-03 and T-04 +# run without the outer guard, so their safety rests on it only reading. +# Changing it needs a new safety review and a new hash here. +payload_sha256=7fd727d79f1eb2e8e3a6f52e25da8f0fe35851b79d262384f6bbc5f5c946ed25 +sum=$( { if command -v sha256sum >/dev/null 2>&1; then sha256sum; else shasum -a 256; fi; } <"$work/payload" | cut -c1-64) +if [ "$sum" != "$payload_sha256" ]; then + echo "HARNESS ERROR: the mutant payload differs from the reviewed one ($sum); refusing to run" >&2 + exit 2 +fi +# The paths substituted into it are plain. +for p in "$work" "$home_marker"; do + case $p in *[!A-Za-z0-9/._-]*) echo "HARNESS ERROR: unexpected character in $p; refusing to run" >&2; exit 2 ;; esac +done + +# read_only : lint (in addition to the hash): every docker command is +# `version` (the xargs case gets it on stdin), and no word of a command that +# changes anything appears (Docker subcommands, HTTP methods other than GET). +# A static check cannot be complete; the hash above is what is relied on. +read_only() { + local words + ! grep -qE '(^|[[:space:]"(=;|&])(rm|rmi|prune|kill|stop|restart|pause|create|run|start|push|pull|build|buildx|builder|load|tag|commit|cp|attach|update|rename|login|logout|volume|network|system|compose|swarm|service|stack|secret|config|plugin|image|container|DELETE|POST|PUT|PATCH|-XDELETE|-XPOST|-XPUT|-d|--data|--json|-F|-T)([[:space:]"'"'"');|&]|$)' "$1" || return 1 + words=$(grep -oE '"?(docker|\$qa_dk|\$qa_p)"?( -H "unix://[^"]*"| -H unix:///var/run/docker.sock| --context canary)? [a-z]+' "$1" | + awk '{print $NF}' | sort -u | tr '\n' ' ') + [ "$words" = "version " ] +} +read_only "$work/payload" || { echo "HARNESS ERROR: the mutant payload fails the read-only lint; refusing to run" >&2; exit 2; } + +# --- T-00 the read-only check itself (payload copies are only read here) +t00_bad=0 +for extra in 'docker image ls' '"docker" rmi x' 'docker system df' 'qa_x=$(docker volume ls)' \ + 'curl -X DELETE --unix-socket "$qa_canary" http://d/images/x' "python3 -c 's.sendall(b\"POST /x\")'" \ + 'docker exec c true' 'docker --context canary info' 'curl -d x --unix-socket "$qa_canary" http://d/x'; do + { cat "$work/payload"; printf '%s\n' "$extra"; } >"$work/payload-t00" + if read_only "$work/payload-t00"; then + t00_bad=$((t00_bad + 1)) + printf 'INFO T-00 accepted: %s\n' "$extra" + fi +done +[ "$t00_bad" -eq 0 ] +result "T-00 the read-only lint rejects a payload with a non-read-only command (9 variants)" $? + +# --- T-01/T-02 no-docker.sh itself +"$root/scripts/ci/no-docker.sh" --check >/dev/null 2>&1 +r=$? +[ "$r" -ne 0 ] +result "T-01 --check fails outside the isolation" $? "(rc $r)" +NO_DOCKER_ACTIVE=1 "$root/scripts/ci/no-docker.sh" touch "$work/t02" >/dev/null 2>&1 +r=$? +[ "$r" -eq 125 ] && [ ! -e "$work/t02" ] +result "T-02 claimed but unproven isolation (NO_DOCKER_ACTIVE outside it): refused, command not run" $? "(rc $r)" + +# --- T-03/T-04 the harness against the mutant +check_safe() { # name + # Refused (125, the mutant never ran) or run isolated (97, every case ran + # and none reached an engine); either way the canary engine got nothing. + local cases answered + cases=$(grep -c ' rc=' "$LOG") + answered=$(grep -c 'canary-engine' "$LOG") + [ "$REQ" -eq 0 ] && [ "$answered" -eq 0 ] + result "$1: no request reached an engine" $? "(canary requests $REQ, answered cases $answered, rc $RC)" + [ "$STARTED" -eq 1 ] + result "$1: the harness was started" $? + if [ "$(uname -s)" = Darwin ] && [ "$t0304" = bare ]; then + # The harness nests its own sandbox inside the guard and runs the mutant. + [ "$RC" -eq 97 ] && [ "$cases" -ge 20 ] + result "$1: the mutant ran to its end inside the harness's isolation" $? "(rc $RC, $cases cases)" + else + # The canary is a reachable engine on Linux: the harness must refuse. + [ "$RC" -eq 125 ] && [ "$cases" -eq 0 ] && + grep -q '^no-docker: refused: inside no-docker.sh, but the isolation check fails' "$OUT" + result "$1: the harness refused before the mutant ran (an engine is reachable)" $? "(rc $RC, $cases cases)" + fi + ! grep -E '^(real-socket-cli|real-socket-curl) rc=0 ' "$LOG" >/dev/null + result "$1: the real Docker socket paths were not reached" $? + [ ! -e "$home_marker" ] + result "$1: nothing written under \$HOME" $? + rm -f "$home_marker" +} +run_case t03 "$harness" devenv "$t0304" +check_safe "T-03 mutant as DEVENV" +run_case t04 "$harness" inplace "$t0304" +check_safe "T-04 mutant edited into bin/devenv" + +# --- T-05 control: the same check catches a harness without containment +run_case t05 "$work/unit-uncontained.sh" inplace guard +answered=$(grep -c 'canary-engine' "$LOG") +[ "$STARTED" -eq 1 ] && [ "$REQ" -gt 0 ] && [ "$answered" -gt 0 ] && [ "$RC" -eq 97 ] +result "T-05 control: without the containment the mutant's commands reach the canary engine" $? "(canary requests $REQ, answered cases $answered, rc $RC)" + +echo +for f in t03 t04 t05; do + printf 'INFO %s cases:\n' "$f" + sed 's/^/INFO /' "$work/$f/mutant.log" 2>/dev/null + printf 'INFO %s harness output (last lines):\n' "$f" + tail -n 8 "$work/$f/out" 2>/dev/null | sed 's/^/INFO /' +done +echo +echo "TOTAL $((pass + fail)) passed=$pass failed=$fail" +[ "$fail" -eq 0 ] && [ "$pass" -gt 0 ] From 9404ee672af34f9675cd2e6613c2494f2f88dcee Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Thu, 29 Oct 2026 07:07:44 +0100 Subject: [PATCH 03/77] docs(dev-environment): describe the Docker isolation of the test suites Documents how the unit and guard suites isolate themselves from Docker on macOS and Linux, what the probe checks, the U-16 check that no longer runs inside the isolation, the regression suite, and the rules for mutation testing: read-only payloads only, never through the real-Docker suites. --- docs/dev-environment.md | 76 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/docs/dev-environment.md b/docs/dev-environment.md index 039abdb..b209bf5 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -413,6 +413,82 @@ $ DEVENV_TEST_PORT_BASE=18080 tests/devenv/integration.sh $ make test-app # application: the four suites below ``` +### Isolation from Docker + +`tests/devenv/unit.sh` and the guard regression suites in +`scripts/ci/tests/` run code under test, including a modified or mutated +copy of it (`DEVENV=`, `GUARD=`, `ADAPTED=` or an edited file). They run it +only where no container engine can be reached: each re-executes itself under +`scripts/ci/no-docker.sh`, so a command in that code cannot touch the +images, containers, volumes, networks or build cache of the machine, however +it reaches Docker (the CLI on `PATH` or by absolute path, a nested shell or +command substitution, `-H`, `--context` or an inherited `DOCKER_HOST`, +`curl --unix-socket` or a raw socket). + +- macOS: `sandbox-exec` denies: + - every connect to a Unix socket; + - TCP beyond loopback, and to loopback ports 22, 2375 and 2376; + - writes under `$HOME`, where Docker Desktop keeps its disk image; + - `open`, `osascript`, `launchctl` and `ssh`, and access to Launch + Services and Apple Events, so that no program is started outside the + sandbox. + + The profile filters these routes; it is not a complete jail. Other + loopback ports stay open for the tests, and the probe asks each listening + port whether it is an engine. +- Linux: `sudo unshare` gives a network namespace with only loopback and an + empty tmpfs over `/run`; the suite then runs as the calling user without + supplementary groups (no `docker` group) and with `no_new_privs`. This + needs passwordless sudo. +- `DOCKER_*` and similar variables are removed and `DOCKER_CONFIG` is an + empty directory. +- Before the suite runs, `scripts/ci/no-docker-probe.py` checks from inside + that the mechanism is in effect and that no engine answers on any endpoint + it finds: the usual socket paths, the Docker contexts, `DOCKER_HOST`, the + sockets under `/run`, `/var/run`, `/tmp`, `$TMPDIR` and `~/.docker`, + ports 2375 and 2376 and every listening loopback port. If the isolation cannot be applied or proven, the + suite does not run (exit 125). +- Setuid programs cannot run inside, so `sudo` is unavailable. The U-16 + check that another local user cannot read a secret therefore prints INFO + and is not counted, in CI as well as locally; the U-16 mode checks still + run. +- On Linux the probe finds an engine socket owned by the calling user only + in the places listed above. + +- macOS applies a sandbox inside another only if the profiles are + identical, so the suites cannot run inside a different sandbox: they are + refused (exit 125). + +`scripts/ci/tests/no-docker.test.sh` checks this with a mutated `bin/devenv` +whose commands reach Docker in more than 20 ways, both as `DEVENV` and edited +into the file. + +- The mutant only reads (`docker version`, `GET /version`); a pre-flight + check refuses to run if it contains anything else. +- Its target is a canary engine of the test, which answers like Docker and + records every request. Two cases name the real socket paths. +- On macOS, `unit.sh` runs on the host as a developer would run it. The + mutant must run to its end with no request reaching the canary and no + answer from the real socket paths. +- On Linux, `unit.sh` runs behind `no-docker.sh`, where the canary is a + reachable engine, so `unit.sh` must refuse. +- The control runs the same mutant under a copy of `unit.sh` without the + re-execution, behind an outer guard that leaves only the canary + reachable. The mutant must reach the canary. + +Mutation testing: + +- Run a mutated copy of `bin/devenv` or of a guard only through these + suites. +- Use payloads that only read or only print. Never use a destructive Docker + command (prune, forced removal, `down -v`, `--rmi`), not even in isolation. +- `tests/devenv/integration.sh` and the `tests/app/` suites use the real + Docker daemon on purpose, so never run them on a mutated copy. + `integration.sh` refuses a `DEVENV` that differs from `bin/devenv`, but no + suite can tell an edited checkout from a mutated one. +- BE-18 mutates a copy of the backend inside the api test container, which + has no Docker socket. + Application suites (`tests/app/`), each in its own Compose projects: - `scan.sh` (SC-xx): Trivy scans of the runtime images (production-like api From 16f1a761a102090037c4823b11c2079b1f14ad92 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Thu, 29 Oct 2026 07:10:57 +0100 Subject: [PATCH 04/77] test(devenv): keep the unit suite's runtime paths out of the checkout U-04 points RUNTIME_DIR, STATE_FILE and LOCK_DIR at the suite's temporary directory but left GATE_DIR, GATE_FILE and SECRETS_DIR at the checkout's .runtime, so write_state in U-05 created (and chmodded) .runtime/gate in the checkout under test. Inside the macOS isolation, which denies writes under $HOME, a fresh checkout therefore failed at U-05. All runtime paths now point at the temporary directory. --- tests/devenv/unit.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/devenv/unit.sh b/tests/devenv/unit.sh index a8041d1..f14eea5 100755 --- a/tests/devenv/unit.sh +++ b/tests/devenv/unit.sh @@ -113,6 +113,10 @@ expect_eq "U-03 all 20 range ports considered" 20 "$#" RUNTIME_DIR=$work/rt STATE_FILE=$RUNTIME_DIR/dev.env LOCK_DIR=$RUNTIME_DIR/lock +# Every runtime path, so that nothing is written into the checkout's .runtime. +SECRETS_DIR=$RUNTIME_DIR/secrets +GATE_DIR=$RUNTIME_DIR/gate +GATE_FILE=$GATE_DIR/security-gate mkdir -p "$RUNTIME_DIR" # Fixtures are written to $work/st first: piping into state_case would run it # in a subshell and lose its pass/fail counts. From e84105a1cf1a4c63dcf781f2b7f404a953efd799 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Thu, 29 Oct 2026 21:20:42 +0100 Subject: [PATCH 05/77] fix(test): close more routes out of the macOS test isolation The macOS profile of scripts/ci/no-docker.sh left routes by which a mutated script could reach the Docker daemon outside the sandbox: - loopback TCP to local services that were already listening (agents, IDE servers, Docker Desktop's published-port forwarders, a socket proxy); the probe only recognised a Docker API answering /_ping; - writes outside $HOME, e.g. into Homebrew's prefix or /Applications (the Docker CLI under Docker.app is user-writable), whose programs run later outside the sandbox; - the exec denials of open, osascript and launchctl match a literal path, and a re-signed copy runs. Deny connects to every loopback port listening before the sandbox starts (listed by the probe's new --listening mode, refused if it cannot be listed), allow writes only under $TMPDIR, /tmp and /dev (refuse if $TMPDIR is under $HOME), and deny job-creation, lsopen and appleevent-send so the routes are closed whatever binary is used. The probe now also proves inside the sandbox that the held ports cannot be connected to and that nothing can be created under $HOME or next to $TMPDIR. --- docs/dev-environment.md | 34 ++++++++++++----- scripts/ci/no-docker-probe.py | 69 ++++++++++++++++++++++++++++++++--- scripts/ci/no-docker.sh | 53 +++++++++++++++++++++------ 3 files changed, 130 insertions(+), 26 deletions(-) diff --git a/docs/dev-environment.md b/docs/dev-environment.md index 6f7bac4..a9adbbe 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -428,14 +428,23 @@ command substitution, `-H`, `--context` or an inherited `DOCKER_HOST`, - macOS: `sandbox-exec` denies: - every connect to a Unix socket; - TCP beyond loopback, and to loopback ports 22, 2375 and 2376; - - writes under `$HOME`, where Docker Desktop keeps its disk image; - - `open`, `osascript`, `launchctl` and `ssh`, and access to Launch - Services and Apple Events, so that no program is started outside the - sandbox. - - The profile filters these routes; it is not a complete jail. Other - loopback ports stay open for the tests, and the probe asks each listening - port whether it is an engine. + - TCP to every loopback port that was already listening when the suite + started, such as a local agent, an IDE, a published container port or + a socket proxy; + - writes anywhere except under `$TMPDIR`, `/tmp` (bash 3.2 writes its + here-documents there) and `/dev`. This covers `$HOME`, where Docker + Desktop keeps its disk image, and places whose programs run outside the + sandbox later, such as Homebrew's prefix and `/Applications`. If + `$TMPDIR` is under `$HOME`, the suite is refused (exit 125); + - `open`, `osascript`, `launchctl` and `ssh`, job submission to launchd, + and access to Launch Services and Apple Events, so that no program is + started outside the sandbox. A re-signed copy of one of these programs + can run, but launchd, Launch Services and Apple Events still refuse it, + and the network routes of `ssh` are denied above. + + The profile filters these routes; it is not a complete jail. Loopback + ports that the suite opens itself stay reachable, and the probe asks each + listening port whether it is an engine. - Linux: `sudo unshare` gives a network namespace with only loopback and an empty tmpfs over `/run`; the suite then runs as the calling user without supplementary groups (no `docker` group) and with `no_new_privs`. This @@ -453,7 +462,10 @@ command substitution, `-H`, `--context` or an inherited `DOCKER_HOST`, and is not counted, in CI as well as locally; the U-16 mode checks still run. - On Linux the probe finds an engine socket owned by the calling user only - in the places listed above. + in the places listed above. Writes are not restricted on Linux, so a + mutant could change files that run later outside the isolation. CI + runners are discarded after each job; on a developer machine, run + mutants only with read-only payloads (see below). - macOS applies a sandbox inside another only if the profiles are identical, so the suites cannot run inside a different sandbox: they are @@ -475,6 +487,10 @@ into the file. - The control runs the same mutant under a copy of `unit.sh` without the re-execution, behind an outer guard that leaves only the canary reachable. The mutant must reach the canary. +- Without a mutant, plain commands inside `no-docker.sh` must not reach a + loopback port that was listening before it started, while a port they + open themselves stays reachable. On macOS they must also be unable to + write under `$HOME` or next to `$TMPDIR`. Mutation testing: diff --git a/scripts/ci/no-docker-probe.py b/scripts/ci/no-docker-probe.py index ca79d25..1be40f9 100644 --- a/scripts/ci/no-docker-probe.py +++ b/scripts/ci/no-docker-probe.py @@ -1,12 +1,17 @@ """Prove that this process cannot reach a container engine. Run by scripts/ci/no-docker.sh inside its isolation, before the command it -protects, and by `no-docker.sh --check`. Exit 0 only if +protects, and by `no-docker.sh --check`. `--listening` only prints the TCP +ports listening on loopback or on all addresses, one per line (no-docker.sh +denies them inside the macOS sandbox). Otherwise exit 0 only if - the isolation mechanism is in effect in this process: macOS: a connect to a Unix socket of our own and a TCP connect to loopback port 2375 and to a non-loopback address are refused by - the sandbox (EPERM), not merely unanswered; + the sandbox (EPERM), not merely unanswered; no loopback port that + was listening before the sandbox started (NO_DOCKER_HELD_PORTS) + can be connected to; and no file can be created under $HOME or + next to $TMPDIR; Linux: no_new_privs is set, no supplementary group is held, the network namespace has only the loopback interface, and /run is a tmpfs without any socket; @@ -102,6 +107,44 @@ def check_mechanism_darwin(): os.rmdir(d) report(denied(socket.AF_INET, ("127.0.0.1", 2375)), "macOS sandbox denies loopback port 2375") report(denied(socket.AF_INET, ("192.0.2.1", 2375)), "macOS sandbox denies non-loopback TCP") + if not ok: + # Not sandboxed: write nothing and connect to nothing else. + return + held = os.environ.get("NO_DOCKER_HELD_PORTS", "").split() + reached = [p for p in held if p.isdigit() and connects(("127.0.0.1", int(p)))] + report(not reached, f"macOS sandbox denies the {len(held)} loopback ports listening before it started" + + (": reached " + " ".join(reached) if reached else "")) + home = os.path.realpath(os.path.expanduser("~")) + tmp = os.path.realpath(os.environ.get("TMPDIR") or "/tmp") + for where, d in (("under $HOME", home), ("outside $TMPDIR", os.path.dirname(tmp))): + report(not_writable(d), f"macOS sandbox denies writes {where} ({d})") + + +def connects(address): + """True if a TCP connect succeeds; nothing is sent.""" + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(1) + try: + s.connect(address) + except OSError: + return False + finally: + s.close() + return True + + +def not_writable(directory): + """True if a new file cannot be created in directory.""" + path = os.path.join(directory, f".no-docker-probe-{os.getpid()}") + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + return False + except OSError: + return True + os.close(fd) + os.unlink(path) + return False def check_mechanism_linux(): @@ -149,13 +192,20 @@ def walk_sockets(roots, limit=20000): pass -def listening_ports(): - """(host, port) of TCP ports listening on loopback or on all addresses.""" +def listening_ports(strict=False): + """(host, port) of TCP ports listening on loopback or on all addresses. + + strict: raise instead of returning an incomplete list (macOS).""" ports = set() if sys.platform == "darwin": try: - out = subprocess.run(["netstat", "-an", "-p", "tcp"], capture_output=True, text=True, timeout=10).stdout + run = subprocess.run(["netstat", "-an", "-p", "tcp"], capture_output=True, text=True, timeout=10) + if run.returncode != 0: + raise OSError(f"netstat exited with {run.returncode}") + out = run.stdout except (OSError, subprocess.SubprocessError): + if strict: + raise out = "" for line in out.splitlines(): f = line.split() @@ -219,6 +269,15 @@ def endpoints(): def main(): + if sys.argv[1:] == ["--listening"]: + # For no-docker.sh, before the isolation: the ports to deny inside it. + try: + ports = listening_ports(strict=True) + except (OSError, subprocess.SubprocessError) as e: + print(f"no-docker: cannot list the listening ports: {e}", file=sys.stderr) + return 1 + print("\n".join(sorted({str(port) for _, port in ports}, key=int))) + return 0 if sys.platform == "darwin": check_mechanism_darwin() elif sys.platform.startswith("linux"): diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh index d18cef3..8da574b 100755 --- a/scripts/ci/no-docker.sh +++ b/scripts/ci/no-docker.sh @@ -13,16 +13,22 @@ # # Isolation: # macOS sandbox-exec: every connect to a Unix socket is denied, as is TCP -# to anything but loopback and to loopback ports 22, 2375 and 2376 -# (ssh and the Docker API ports); nothing under $HOME can be -# written (Docker Desktop keeps its disk image there); open, -# osascript, launchctl and ssh cannot run and Launch Services and -# Apple Events cannot be reached (the known ways to have a program -# started outside the sandbox). The profile filters these routes; -# it is not a complete jail. Setuid programs cannot run in the sandbox, so -# ps is an unprivileged, ad-hoc signed copy. Loopback TCP to other -# ports stays open for the tests; the probe asks every listening -# port whether it is an engine (a socket proxy, for example). +# to anything but loopback, to loopback ports 22, 2375 and 2376 +# (ssh and the Docker API ports) and to every loopback port that +# was listening before the command started (local services: an +# agent, an IDE, a published container port or a socket proxy); +# nothing can be written except under $TMPDIR, /tmp and /dev (not +# $HOME, where Docker Desktop keeps its disk image, nor Homebrew's +# prefix or /Applications, whose programs run outside the sandbox +# later); +# open, osascript, launchctl and ssh cannot run, and jobs cannot be +# submitted to launchd, nor Launch Services or Apple Events used +# (the known ways to have a program started outside the sandbox). +# The profile filters these routes; it is not a complete jail. +# Setuid programs cannot run in the sandbox, so ps is an +# unprivileged, ad-hoc signed copy. Loopback ports the command +# opens itself stay reachable for the tests; the probe asks every +# listening port whether it is an engine. # Linux sudo unshare: a network namespace with only loopback and a mount # namespace with an empty tmpfs over /run (and /var/run), then the # calling user without supplementary groups (no docker group) and @@ -91,15 +97,38 @@ case $(uname -s) in for i in "${!envargs[@]}"; do case ${envargs[$i]} in PATH=*) envargs[i]="PATH=$work/bin:${envargs[$i]#PATH=}" ;; esac done - home=$(cd -P "$HOME" && pwd -P) + home=$(cd -P "$HOME" && pwd -P) || refuse "cannot resolve \$HOME" + tmp=$(cd -P "${TMPDIR:-/tmp}" && pwd -P) || refuse "cannot resolve the temporary directory" + # The command may write only under $TMPDIR, /tmp (bash 3.2 writes its + # here-documents there whatever $TMPDIR is) and /dev: programs outside + # the sandbox run files from $HOME, Homebrew's prefix or /Applications. + case $tmp/ in + / | "$home"/*) refuse "the temporary directory $tmp is / or under \$HOME" ;; + esac + case $tmp in *[!A-Za-z0-9/._-]*) refuse "unexpected character in the temporary directory $tmp" ;; esac + # Local services that were listening before the command started (an + # agent, an IDE, a published container port or a socket proxy) are not + # the command's: it may connect only to loopback ports it opens itself. + listening=$(python3 -I "$probe" --listening) || refuse "cannot list the listening TCP ports" + held="" + for port in $listening; do + case $port in *[!0-9]* | "") refuse "unexpected port $port" ;; esac + held="$held (remote ip \"localhost:$port\")" + done + envargs+=("NO_DOCKER_HELD_PORTS=$(printf '%s' "$listening" | tr '\n' ' ')") profile="(version 1) (allow default) (deny network-outbound (remote unix-socket)) (deny network-outbound (remote ip)) (allow network-outbound (remote ip \"localhost:*\")) -(deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")) +(deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")$held) +(deny file-write*) +(allow file-write* (subpath \"$tmp\") (subpath \"/private/tmp\") (subpath \"/dev\")) (deny file-write* (subpath \"$home\")) (deny process-exec (literal \"/usr/bin/open\") (literal \"/usr/bin/osascript\") (literal \"/bin/launchctl\") (literal \"/usr/bin/ssh\")) +(deny job-creation) +(deny lsopen) +(deny appleevent-send) (deny mach-lookup (global-name \"com.apple.coreservices.launchservicesd\") (global-name \"com.apple.coreservices.appleevents\") (global-name-regex #\"^com\\.apple\\.lsd\\.\"))" sandbox-exec -p "$profile" /usr/bin/env -i "${envargs[@]}" "$BASH" "$here/no-docker.sh" --inner "$@" ;; From 6222b6a082f9629dc360206a9e653ee959ebcd6c Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Thu, 29 Oct 2026 21:20:49 +0100 Subject: [PATCH 06/77] test(ci): check the write limits and held loopback ports of the isolation T-06 (macOS) runs a plain command inside no-docker.sh that tries to create a file under $HOME and next to $TMPDIR and one in the suite's own directory: only the last may exist. T-07 starts a loopback listener before the isolation and requires that a command inside it cannot reach it while a port it opens itself stays reachable. Neither case uses the mutant; both fail against the previous profile. --- scripts/ci/tests/no-docker.test.sh | 54 +++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/scripts/ci/tests/no-docker.test.sh b/scripts/ci/tests/no-docker.test.sh index 8c6ec7f..6d1a714 100755 --- a/scripts/ci/tests/no-docker.test.sh +++ b/scripts/ci/tests/no-docker.test.sh @@ -26,6 +26,9 @@ # only if it is identical, so the harness's own isolation cannot be # tested behind the guard. Should that isolation fail, the read-only # payload would reach the real daemon, which the cases detect. +# T-06 and T-07 run plain commands, no mutant, inside no-docker.sh: they +# check the write limits (macOS) and that loopback ports listening before +# the isolation started cannot be reached from inside it. # This suite itself runs no docker command. # # Usage: no-docker.test.sh @@ -51,7 +54,11 @@ home=$(cd -P "$HOME" && pwd -P) nonce=$$-$(date +%s) canary=$work/canary.sock home_marker=$home/.flowcrm-no-docker-test-$nonce -trap 'rm -f "$home_marker"; rm -rf "$work"' EXIT +# Next to $TMPDIR: writable by the user, outside what the sandbox allows. +tmp=$(cd -P "${TMPDIR:-/tmp}" && pwd -P) +[ -n "$tmp" ] && [ "$tmp" != / ] || exit 2 +outside_marker=$(dirname "$tmp")/.flowcrm-no-docker-test-$nonce +trap 'rm -f "$home_marker" "$outside_marker"; rm -rf "$work"' EXIT pass=0 fail=0 @@ -307,6 +314,51 @@ r=$? [ "$r" -eq 125 ] && [ ! -e "$work/t02" ] result "T-02 claimed but unproven isolation (NO_DOCKER_ACTIVE outside it): refused, command not run" $? "(rc $r)" +# --- T-06/T-07 plain commands (no mutant) inside no-docker.sh +if [ "$(uname -s)" = Darwin ]; then + "$root/scripts/ci/no-docker.sh" /bin/sh -c 'touch "$1" "$2" 2>/dev/null; : >"$3"' _ \ + "$home_marker" "$outside_marker" "$work/t06" >/dev/null 2>&1 + r=$? + [ "$r" -eq 0 ] && [ -e "$work/t06" ] && [ ! -e "$home_marker" ] && [ ! -e "$outside_marker" ] + result "T-06 writes only under \$TMPDIR: not under \$HOME, not next to \$TMPDIR" $? "(rc $r)" + rm -f "$home_marker" "$outside_marker" +else + echo "INFO T-06 skipped: on Linux writes are not restricted (see docs/dev-environment.md)" +fi +cat >"$work/t07.py" <<'PY' +import socket, sys, time +s = socket.socket() +s.bind(("127.0.0.1", 0)) +s.listen(4) +open(sys.argv[1], "w").write(str(s.getsockname()[1])) +time.sleep(60) +PY +python3 -I "$work/t07.py" "$work/t07.port" & +t07pid=$! +for _ in 1 2 3 4 5 6 7 8 9 10; do [ -s "$work/t07.port" ] && break; sleep 0.2; done +t07=$("$root/scripts/ci/no-docker.sh" python3 -I -c ' +import socket, sys +def reach(port): + s = socket.socket() + s.settimeout(2) + try: + s.connect(("127.0.0.1", port)) + return "reached" + except OSError: + return "refused" + finally: + s.close() +own = socket.socket() +own.bind(("127.0.0.1", 0)) +own.listen(1) +print(reach(int(sys.argv[1])), reach(own.getsockname()[1])) +' "$(cat "$work/t07.port" 2>/dev/null || echo 0)" 2>/dev/null) +r=$? +kill "$t07pid" 2>/dev/null +wait "$t07pid" 2>/dev/null +[ "$r" -eq 0 ] && [ -s "$work/t07.port" ] && [ "$t07" = "refused reached" ] +result "T-07 a loopback port listening before the isolation is unreachable, one opened inside it is reachable" $? "(rc $r: $t07)" + # --- T-03/T-04 the harness against the mutant check_safe() { # name # Refused (125, the mutant never ran) or run isolated (97, every case ran From 3d2dbe360a9eef153d2cbed6f6b821182961a3b4 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Thu, 29 Oct 2026 22:13:38 +0100 Subject: [PATCH 07/77] fix(test): narrow the macOS isolation's /tmp, port list and side channels Follow-ups from the safety review of the previous isolation change: - The held-port list is accepted only if netstat succeeded, printed its column header and every LISTEN row parsed; otherwise the command is refused (125). An empty list is thus provably empty. - /tmp is no longer writable, except bash 3.2's sh-thd* here-document files, so agent scratchpads and other tools' files there cannot be planted. $TMPDIR must be a directory of its own: /, /tmp, under /tmp or under $HOME is refused, which also removes the vacuous TMPDIR=/tmp write check. The probe now also proves /tmp is write-denied. - Deny user-preference-write: defaults write reached ~/Library/Preferences through cfprefsd although $HOME is write-denied (measured). - Deny the TIOCSTI ioctl: a command run from an interactive terminal could inject keystrokes into the shell that started it (measured). Hard links from $TMPDIR to a write-denied file were measured to be refused already. The docs list the measured and the residual routes. --- docs/dev-environment.md | 49 ++++++++++++++++++++++++++++------- scripts/ci/no-docker-probe.py | 27 +++++++++++++------ scripts/ci/no-docker.sh | 28 +++++++++++++------- 3 files changed, 77 insertions(+), 27 deletions(-) diff --git a/docs/dev-environment.md b/docs/dev-environment.md index a9adbbe..15e3ae3 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -430,21 +430,47 @@ command substitution, `-H`, `--context` or an inherited `DOCKER_HOST`, - TCP beyond loopback, and to loopback ports 22, 2375 and 2376; - TCP to every loopback port that was already listening when the suite started, such as a local agent, an IDE, a published container port or - a socket proxy; - - writes anywhere except under `$TMPDIR`, `/tmp` (bash 3.2 writes its - here-documents there) and `/dev`. This covers `$HOME`, where Docker - Desktop keeps its disk image, and places whose programs run outside the - sandbox later, such as Homebrew's prefix and `/Applications`. If - `$TMPDIR` is under `$HOME`, the suite is refused (exit 125); + a socket proxy. The list comes from `netstat`. If `netstat` fails, + prints no column header or prints a LISTEN row that cannot be parsed, + the suite is refused (exit 125). An empty list is accepted only when + `netstat` printed its header and no LISTEN row; + - writes anywhere except under `$TMPDIR` and `/dev`, and in `/tmp` the + `sh-thd*` here-document files of bash 3.2, which it creates there + whatever `$TMPDIR` is. This covers `$HOME`, where Docker Desktop keeps + its disk image, and places whose files are run or read outside the + sandbox later: Homebrew's prefix, `/Applications` and other tools' + directories in `/tmp`, such as agent scratchpads. `$TMPDIR` must + therefore be a directory of its own. If it is `/`, `/tmp`, under `/tmp` + or under `$HOME`, the suite is refused (exit 125). The macOS default + under `/var/folders` is fine; + - hard links do not get around this. Measured: a link from `$TMPDIR` to + a file in a write-denied place is refused; + - preference writes through `cfprefsd` (`defaults write`), which would + otherwise reach `~/Library/Preferences` although `$HOME` is + write-denied; + - `TIOCSTI` on the terminal, which would otherwise let a command inject + keystrokes into the shell that started the suite; - `open`, `osascript`, `launchctl` and `ssh`, job submission to launchd, and access to Launch Services and Apple Events, so that no program is started outside the sandbox. A re-signed copy of one of these programs can run, but launchd, Launch Services and Apple Events still refuse it, and the network routes of `ssh` are denied above. - The profile filters these routes; it is not a complete jail. Loopback - ports that the suite opens itself stay reachable, and the probe asks each - listening port whether it is an engine. + The profile filters these routes; it is not a complete jail. Known + residual routes: + - loopback ports that the suite opens itself stay reachable. So does a + service that starts listening after the suite started; the probe asks + each listening port whether it is an engine only at the start; + - a command can still create `sh-thd*` files or directories in `/tmp`. + bash creates its own here-document files exclusively, so at worst a + later here-document of another shell fails; + - signals to the user's processes outside the sandbox are allowed. A + command could stop Docker Desktop, but not change its objects; + - login items and background tasks are registered through Launch + Services, Apple Events or launchd, which are denied. Their other + interfaces were not measured; + - the write check next to `$TMPDIR` proves nothing when the directory + holding `$TMPDIR` is not writable by the user anyway. - Linux: `sudo unshare` gives a network namespace with only loopback and an empty tmpfs over `/run`; the suite then runs as the calling user without supplementary groups (no `docker` group) and with `no_new_privs`. This @@ -490,7 +516,10 @@ into the file. - Without a mutant, plain commands inside `no-docker.sh` must not reach a loopback port that was listening before it started, while a port they open themselves stays reachable. On macOS they must also be unable to - write under `$HOME` or next to `$TMPDIR`. + write under `$HOME`, next to `$TMPDIR` or in `/tmp`, while bash 3.2 + here-documents still work. They must not be able to write a preference + or use `TIOCSTI` either. An incomplete port list and `TMPDIR=/tmp` must + be refused. Mutation testing: diff --git a/scripts/ci/no-docker-probe.py b/scripts/ci/no-docker-probe.py index 1be40f9..53114da 100644 --- a/scripts/ci/no-docker-probe.py +++ b/scripts/ci/no-docker-probe.py @@ -10,8 +10,8 @@ loopback port 2375 and to a non-loopback address are refused by the sandbox (EPERM), not merely unanswered; no loopback port that was listening before the sandbox started (NO_DOCKER_HELD_PORTS) - can be connected to; and no file can be created under $HOME or - next to $TMPDIR; + can be connected to; and no file can be created under $HOME, next + to $TMPDIR or in /tmp; Linux: no_new_privs is set, no supplementary group is held, the network namespace has only the loopback interface, and /run is a tmpfs without any socket; @@ -116,7 +116,7 @@ def check_mechanism_darwin(): + (": reached " + " ".join(reached) if reached else "")) home = os.path.realpath(os.path.expanduser("~")) tmp = os.path.realpath(os.environ.get("TMPDIR") or "/tmp") - for where, d in (("under $HOME", home), ("outside $TMPDIR", os.path.dirname(tmp))): + for where, d in (("under $HOME", home), ("outside $TMPDIR", os.path.dirname(tmp)), ("in /tmp", "/private/tmp")): report(not_writable(d), f"macOS sandbox denies writes {where} ({d})") @@ -195,7 +195,10 @@ def walk_sockets(roots, limit=20000): def listening_ports(strict=False): """(host, port) of TCP ports listening on loopback or on all addresses. - strict: raise instead of returning an incomplete list (macOS).""" + strict (macOS): raise instead of returning a list that may be + incomplete. The list is accepted, even empty, only if netstat succeeded, + printed its column header and every LISTEN row has a parsable local + address.""" ports = set() if sys.platform == "darwin": try: @@ -203,16 +206,24 @@ def listening_ports(strict=False): if run.returncode != 0: raise OSError(f"netstat exited with {run.returncode}") out = run.stdout + if strict and not any(line.split()[:4] == ["Proto", "Recv-Q", "Send-Q", "Local"] + for line in out.splitlines()): + raise OSError("netstat printed no column header") except (OSError, subprocess.SubprocessError): if strict: raise out = "" for line in out.splitlines(): f = line.split() - if len(f) >= 6 and f[-1] == "LISTEN": - host, _, port = f[3].rpartition(".") - if host in ("127.0.0.1", "::1", "*") and port.isdigit(): - ports.add(("::1" if host == "::1" else "127.0.0.1", int(port))) + if not f or f[-1] != "LISTEN": + continue + host, _, port = f[3].rpartition(".") if len(f) >= 6 else ("", "", "") + if not host or not port.isdigit(): + if strict: + raise OSError(f"unparsable netstat LISTEN row: {line.strip()}") + continue + if host in ("127.0.0.1", "::1", "*"): + ports.add(("::1" if host == "::1" else "127.0.0.1", int(port))) else: for name in ("/proc/net/tcp", "/proc/net/tcp6"): try: diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh index 8da574b..c800905 100755 --- a/scripts/ci/no-docker.sh +++ b/scripts/ci/no-docker.sh @@ -17,10 +17,14 @@ # (ssh and the Docker API ports) and to every loopback port that # was listening before the command started (local services: an # agent, an IDE, a published container port or a socket proxy); -# nothing can be written except under $TMPDIR, /tmp and /dev (not -# $HOME, where Docker Desktop keeps its disk image, nor Homebrew's -# prefix or /Applications, whose programs run outside the sandbox -# later); +# the list of those ports must be complete, or the command is +# refused; nothing can be written except under $TMPDIR, /dev and +# bash 3.2's here-document files in /tmp (not $HOME, where Docker +# Desktop keeps its disk image, nor Homebrew's prefix, +# /Applications or other tools' files in /tmp, which run or are +# read outside the sandbox later), so $TMPDIR must not be /tmp or +# under it or under $HOME; no preference can be written through +# cfprefsd and no keystroke injected into the terminal (TIOCSTI); # open, osascript, launchctl and ssh cannot run, and jobs cannot be # submitted to launchd, nor Launch Services or Apple Events used # (the known ways to have a program started outside the sandbox). @@ -99,11 +103,15 @@ case $(uname -s) in done home=$(cd -P "$HOME" && pwd -P) || refuse "cannot resolve \$HOME" tmp=$(cd -P "${TMPDIR:-/tmp}" && pwd -P) || refuse "cannot resolve the temporary directory" - # The command may write only under $TMPDIR, /tmp (bash 3.2 writes its - # here-documents there whatever $TMPDIR is) and /dev: programs outside - # the sandbox run files from $HOME, Homebrew's prefix or /Applications. + # The command may write only under $TMPDIR and /dev, and in /tmp only + # the here-document files of bash 3.2 (sh-thd*, which it creates there + # whatever $TMPDIR is): programs outside the sandbox run files from + # $HOME, Homebrew's prefix, /Applications or other tools' directories in + # /tmp (agent scratchpads, for example). So $TMPDIR must be a directory + # of its own: not /, not under $HOME and not /tmp or under it. case $tmp/ in - / | "$home"/*) refuse "the temporary directory $tmp is / or under \$HOME" ;; + / | "$home"/* | /private/tmp/* | /tmp/*) + refuse "the temporary directory $tmp is /, /tmp or under /tmp or \$HOME; set TMPDIR to a private directory (the macOS default under /var/folders)" ;; esac case $tmp in *[!A-Za-z0-9/._-]*) refuse "unexpected character in the temporary directory $tmp" ;; esac # Local services that were listening before the command started (an @@ -123,8 +131,10 @@ case $(uname -s) in (allow network-outbound (remote ip \"localhost:*\")) (deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")$held) (deny file-write*) -(allow file-write* (subpath \"$tmp\") (subpath \"/private/tmp\") (subpath \"/dev\")) +(allow file-write* (subpath \"$tmp\") (subpath \"/dev\") (literal \"/private/tmp\") (regex #\"^/private/tmp/sh-thd[^/]*\$\")) (deny file-write* (subpath \"$home\")) +(deny user-preference-write) +(deny file-ioctl (ioctl-command #x80017472)) (deny process-exec (literal \"/usr/bin/open\") (literal \"/usr/bin/osascript\") (literal \"/bin/launchctl\") (literal \"/usr/bin/ssh\")) (deny job-creation) (deny lsopen) From cb743813942a3806a725537cdc5eb34f15668cb2 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Thu, 29 Oct 2026 22:13:38 +0100 Subject: [PATCH 08/77] test(ci): cover the port-list, /tmp, TMPDIR, preference and TIOCSTI limits Plain commands inside no-docker.sh, no mutant (macOS): - T-06 now also requires that /tmp is write-denied while a bash 3.2 here-document still works; - T-08: a fake netstat without its header or with an unparsable LISTEN row is refused (125, command not run); header and no LISTEN row is an accepted empty list; - T-09: TMPDIR=/tmp is refused; - T-10: defaults write changes no preference, and TIOCSTI on the controlling terminal of a pseudo-terminal is denied. Against the previous profile, T-06, T-08 (two cases), T-09 and both T-10 cases fail. --- scripts/ci/tests/no-docker.test.sh | 73 ++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/scripts/ci/tests/no-docker.test.sh b/scripts/ci/tests/no-docker.test.sh index 6d1a714..308657b 100755 --- a/scripts/ci/tests/no-docker.test.sh +++ b/scripts/ci/tests/no-docker.test.sh @@ -26,9 +26,10 @@ # only if it is identical, so the harness's own isolation cannot be # tested behind the guard. Should that isolation fail, the read-only # payload would reach the real daemon, which the cases detect. -# T-06 and T-07 run plain commands, no mutant, inside no-docker.sh: they -# check the write limits (macOS) and that loopback ports listening before -# the isolation started cannot be reached from inside it. +# T-06 to T-10 run plain commands, no mutant, inside no-docker.sh: they +# check the write limits, the complete held-port list, the refusal of a +# shared $TMPDIR, preferences and TIOCSTI (macOS), and that loopback ports +# listening before the isolation started cannot be reached from inside it. # This suite itself runs no docker command. # # Usage: no-docker.test.sh @@ -58,7 +59,8 @@ home_marker=$home/.flowcrm-no-docker-test-$nonce tmp=$(cd -P "${TMPDIR:-/tmp}" && pwd -P) [ -n "$tmp" ] && [ "$tmp" != / ] || exit 2 outside_marker=$(dirname "$tmp")/.flowcrm-no-docker-test-$nonce -trap 'rm -f "$home_marker" "$outside_marker"; rm -rf "$work"' EXIT +tmp_marker=/private/tmp/.flowcrm-no-docker-test-$nonce +trap 'rm -f "$home_marker" "$outside_marker" "$tmp_marker"; rm -rf "$work"' EXIT pass=0 fail=0 @@ -316,14 +318,65 @@ result "T-02 claimed but unproven isolation (NO_DOCKER_ACTIVE outside it): refus # --- T-06/T-07 plain commands (no mutant) inside no-docker.sh if [ "$(uname -s)" = Darwin ]; then - "$root/scripts/ci/no-docker.sh" /bin/sh -c 'touch "$1" "$2" 2>/dev/null; : >"$3"' _ \ - "$home_marker" "$outside_marker" "$work/t06" >/dev/null 2>&1 + "$root/scripts/ci/no-docker.sh" /bin/bash -c 'touch "$1" "$2" "$3" 2>/dev/null; cat >"$4" </dev/null 2>&1 r=$? - [ "$r" -eq 0 ] && [ -e "$work/t06" ] && [ ! -e "$home_marker" ] && [ ! -e "$outside_marker" ] - result "T-06 writes only under \$TMPDIR: not under \$HOME, not next to \$TMPDIR" $? "(rc $r)" - rm -f "$home_marker" "$outside_marker" + [ "$r" -eq 0 ] && [ "$(cat "$work/t06" 2>/dev/null)" = here-document ] && + [ ! -e "$home_marker" ] && [ ! -e "$outside_marker" ] && [ ! -e "$tmp_marker" ] + result "T-06 writes only under \$TMPDIR (bash 3.2 here-documents work): not under \$HOME, next to \$TMPDIR or in /tmp" $? "(rc $r)" + rm -f "$home_marker" "$outside_marker" "$tmp_marker" + + # T-08: the held-port list must be complete. A netstat without its column + # header, or with a LISTEN row that cannot be parsed, refuses; one with + # the header and no LISTEN row is a provably empty list. + mkdir -p "$work/t08" + t08() { # name netstat-output expected-rc + printf '#!/bin/sh\nprintf "%%b" %s\n' "'$2'" >"$work/t08/netstat" + chmod +x "$work/t08/netstat" + rm -f "$work/t08/ran" + PATH=$work/t08:$PATH "$root/scripts/ci/no-docker.sh" /usr/bin/touch "$work/t08/ran" >/dev/null 2>&1 + r=$? + if [ "$3" -eq 0 ]; then + [ "$r" -eq 0 ] && [ -e "$work/t08/ran" ] + else + [ "$r" -eq 125 ] && [ ! -e "$work/t08/ran" ] + fi + result "T-08 $1" $? "(rc $r)" + } + hdr='Active Internet connections (including servers)\nProto Recv-Q Send-Q Local Address Foreign Address (state)\n' + t08 "netstat without its column header: refused, command not run" 'tcp4 0 0 127.0.0.1.1 *.* LISTEN\n' 125 + t08 "an unparsable LISTEN row: refused, command not run" "${hdr}tcp4 0 0 garbage *.* LISTEN\n" 125 + t08 "header and no LISTEN row: an empty list is accepted" "${hdr}tcp4 0 0 127.0.0.1.1 127.0.0.1.2 ESTABLISHED\n" 0 + + # T-09: $TMPDIR must be a directory of its own, not /tmp. + TMPDIR=/tmp "$root/scripts/ci/no-docker.sh" /usr/bin/touch "$work/t09" >/dev/null 2>&1 + r=$? + [ "$r" -eq 125 ] && [ ! -e "$work/t09" ] + result "T-09 TMPDIR=/tmp: refused, command not run" $? "(rc $r)" + + # T-10: no preference written through cfprefsd, no keystroke injected + # into the terminal (TIOCSTI), run in a pseudo-terminal of its own. + t10_domain=com.flowcrm.no-docker-test-$nonce + "$root/scripts/ci/no-docker.sh" /usr/bin/defaults write "$t10_domain" k v >/dev/null 2>&1 + t10_pref=$(defaults read "$t10_domain" k 2>/dev/null) + defaults delete "$t10_domain" >/dev/null 2>&1 + rm -f "$home/Library/Preferences/$t10_domain.plist" + [ -z "$t10_pref" ] + result "T-10 defaults write inside the isolation changes no preference" $? + t10=$(script -q /dev/null "$root/scripts/ci/no-docker.sh" python3 -I -c ' +import fcntl, os +fd = os.open("/dev/tty", os.O_RDWR) +try: + fcntl.ioctl(fd, 0x80017472, b" ") + print("t10-injected") +except OSError as e: + print("t10-denied", e.errno) +' /dev/null | tr -d '\r') + case $t10 in *"t10-denied 1"*) r=0 ;; *) r=1 ;; esac + result "T-10 TIOCSTI on the controlling terminal is denied" $r "($(printf '%s' "$t10" | grep t10-))" else - echo "INFO T-06 skipped: on Linux writes are not restricted (see docs/dev-environment.md)" + echo "INFO T-06, T-08, T-09, T-10 skipped: macOS sandbox profile only (on Linux writes are not restricted, see docs/dev-environment.md)" fi cat >"$work/t07.py" <<'PY' import socket, sys, time From 38b5ebd1bd6ba43c679688d4edca8f2f33200a88 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Thu, 29 Oct 2026 22:24:51 +0100 Subject: [PATCH 09/77] fix(test): describe the services and temporary files of the isolation neutrally Reword the comments of scripts/ci/no-docker.sh and the docs that named the local services and the other programs' files in /tmp the isolation protects, using generic examples. --- docs/dev-environment.md | 12 ++++++------ scripts/ci/no-docker.sh | 18 ++++++++++-------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/dev-environment.md b/docs/dev-environment.md index 15e3ae3..318233d 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -429,17 +429,17 @@ command substitution, `-H`, `--context` or an inherited `DOCKER_HOST`, - every connect to a Unix socket; - TCP beyond loopback, and to loopback ports 22, 2375 and 2376; - TCP to every loopback port that was already listening when the suite - started, such as a local agent, an IDE, a published container port or - a socket proxy. The list comes from `netstat`. If `netstat` fails, - prints no column header or prints a LISTEN row that cannot be parsed, - the suite is refused (exit 125). An empty list is accepted only when + started, such as a local development server, an IDE, a published + container port or a socket proxy. The list comes from `netstat`. If + `netstat` fails, prints no column header or prints a LISTEN row that + cannot be parsed, the suite is refused (exit 125). An empty list is accepted only when `netstat` printed its header and no LISTEN row; - writes anywhere except under `$TMPDIR` and `/dev`, and in `/tmp` the `sh-thd*` here-document files of bash 3.2, which it creates there whatever `$TMPDIR` is. This covers `$HOME`, where Docker Desktop keeps its disk image, and places whose files are run or read outside the - sandbox later: Homebrew's prefix, `/Applications` and other tools' - directories in `/tmp`, such as agent scratchpads. `$TMPDIR` must + sandbox later: Homebrew's prefix, `/Applications` and other programs' + temporary files in `/tmp`. `$TMPDIR` must therefore be a directory of its own. If it is `/`, `/tmp`, under `/tmp` or under `$HOME`, the suite is refused (exit 125). The macOS default under `/var/folders` is fine; diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh index c800905..9965bbc 100755 --- a/scripts/ci/no-docker.sh +++ b/scripts/ci/no-docker.sh @@ -15,8 +15,9 @@ # macOS sandbox-exec: every connect to a Unix socket is denied, as is TCP # to anything but loopback, to loopback ports 22, 2375 and 2376 # (ssh and the Docker API ports) and to every loopback port that -# was listening before the command started (local services: an -# agent, an IDE, a published container port or a socket proxy); +# was listening before the command started (local services: a +# development server, an IDE, a published container port or a +# socket proxy); # the list of those ports must be complete, or the command is # refused; nothing can be written except under $TMPDIR, /dev and # bash 3.2's here-document files in /tmp (not $HOME, where Docker @@ -106,17 +107,18 @@ case $(uname -s) in # The command may write only under $TMPDIR and /dev, and in /tmp only # the here-document files of bash 3.2 (sh-thd*, which it creates there # whatever $TMPDIR is): programs outside the sandbox run files from - # $HOME, Homebrew's prefix, /Applications or other tools' directories in - # /tmp (agent scratchpads, for example). So $TMPDIR must be a directory - # of its own: not /, not under $HOME and not /tmp or under it. + # $HOME, Homebrew's prefix, /Applications or other programs' temporary + # files in /tmp. So $TMPDIR must be a directory of its own: not /, not + # under $HOME and not /tmp or under it. case $tmp/ in / | "$home"/* | /private/tmp/* | /tmp/*) refuse "the temporary directory $tmp is /, /tmp or under /tmp or \$HOME; set TMPDIR to a private directory (the macOS default under /var/folders)" ;; esac case $tmp in *[!A-Za-z0-9/._-]*) refuse "unexpected character in the temporary directory $tmp" ;; esac - # Local services that were listening before the command started (an - # agent, an IDE, a published container port or a socket proxy) are not - # the command's: it may connect only to loopback ports it opens itself. + # Local services that were listening before the command started (a + # development server, an IDE, a published container port or a socket + # proxy) are not the command's: it may connect only to loopback ports it + # opens itself. listening=$(python3 -I "$probe" --listening) || refuse "cannot list the listening TCP ports" held="" for port in $listening; do From 8a5a676f5049e967818296cb2a80619739e7bd5e Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 09:01:48 +0100 Subject: [PATCH 10/77] fix(test): isolate Linux runs in pid, IPC and read-only mount namespaces and narrow the macOS sandbox Linux is supported on GitHub-hosted runners only (GITHUB_ACTIONS, RUNNER_ENVIRONMENT, a Runner.Worker ancestor, workspace and RUNNER_TEMP under $HOME) and uid 0 is refused everywhere. The namespace adds --pid --fork --kill-child --mount-proc --ipc under a root pid 1 that waits and reaps; tmpfs over /run, /var/run, /tmp, /var/tmp and /dev/shm, a fresh /dev/mqueue, a masked /sys/fs/cgroup, every other mount remounted read-only; setpriv drops groups and all capabilities with no_new_privs; setsid --wait; an allowlisted environment; workflow commands are stopped around the run. Unix sockets are listed outside within a time budget and none may accept a connection inside. On macOS $TMPDIR must be the per-user directory under /private/var/folders, writes are limited to a fresh run directory (the command's $TMPDIR) and a measured set of /dev nodes, signals stay within the sandbox, and the login item, background task and Docker Desktop services cannot be looked up. --work DIR makes a fresh, empty directory of the caller the run directory and keeps it; unit.sh passes NO_DOCKER_WORK on as --work. The probe checks each property from inside and fails closed; every listening port number is held; a truncated socket walk fails on Linux. --- scripts/ci/no-docker-probe.py | 532 +++++++++++++++++++++++++++------- scripts/ci/no-docker.sh | 355 +++++++++++++++++------ tests/devenv/unit.sh | 3 +- 3 files changed, 704 insertions(+), 186 deletions(-) diff --git a/scripts/ci/no-docker-probe.py b/scripts/ci/no-docker-probe.py index 53114da..3a159f9 100644 --- a/scripts/ci/no-docker-probe.py +++ b/scripts/ci/no-docker-probe.py @@ -1,46 +1,68 @@ -"""Prove that this process cannot reach a container engine. - -Run by scripts/ci/no-docker.sh inside its isolation, before the command it -protects, and by `no-docker.sh --check`. `--listening` only prints the TCP -ports listening on loopback or on all addresses, one per line (no-docker.sh -denies them inside the macOS sandbox). Otherwise exit 0 only if - - - the isolation mechanism is in effect in this process: - macOS: a connect to a Unix socket of our own and a TCP connect to - loopback port 2375 and to a non-loopback address are refused by - the sandbox (EPERM), not merely unanswered; no loopback port that - was listening before the sandbox started (NO_DOCKER_HELD_PORTS) - can be connected to; and no file can be created under $HOME, next - to $TMPDIR or in /tmp; - Linux: no_new_privs is set, no supplementary group is held, the - network namespace has only the loopback interface, and /run is a - tmpfs without any socket; +"""Prove that this process is isolated by scripts/ci/no-docker.sh. + +Run by no-docker.sh inside its isolation, before the command it protects, +and by `no-docker.sh --check`. Exit 0 only if + + - every mechanism of the isolation is in effect in this process: + macOS: a connect to a Unix socket of our own, a TCP connect to + loopback port 2375 and to a non-loopback address are refused by the + sandbox (EPERM), not merely unanswered; no loopback port that was + listening before the sandbox started (NO_DOCKER_HELD_PORTS) can be + connected to; no signal reaches no-docker.sh outside the sandbox + (NO_DOCKER_OUTER_PID, EPERM); no file can be created under $HOME, in + the user's temporary directory (NO_DOCKER_USER_TMP), in /tmp or in + /private/var/tmp, while $TMPDIR (inside the run's directory) is + writable; + Linux: not uid 0, no capability in any set, no_new_privs, no + supplementary group; network, mount, pid and IPC namespaces that + differ from the caller's (NO_DOCKER_HOST_NS); a pid namespace below + the caller's whose pid 1 is the root shell of no-docker.sh and whose + other processes are all ours; a session of our own without a + controlling terminal; only the loopback interface; /tmp, /var/tmp, + /dev/shm, /run and /dev/mqueue are new file systems + (NO_DOCKER_HOST_DEVS) and /sys/fs/cgroup is masked; every other + mount reachable by path is read-only; writes to $HOME, + $GITHUB_WORKSPACE and $RUNNER_TEMP (NO_DOCKER_PROTECTED) fail; no + cgroup file is writable; no socket in the private directories; and + no Unix socket that no-docker.sh listed outside (NO_DOCKER_SOCKETS) + accepts a connection; - and no container engine answers on any endpoint we can find: the usual socket paths of Docker Engine, Docker Desktop, rootless Docker, Podman, Colima, OrbStack and Rancher Desktop; every endpoint of the Docker contexts under ~/.docker; DOCKER_HOST and the DOCKER_HOST the - caller of no-docker.sh had (NO_DOCKER_ORIGINAL_HOST); every socket under /run, /var/run, /tmp, - $TMPDIR and ~/.docker; loopback ports 2375 and 2376; and every TCP - port listening on loopback or on all addresses (a socket proxy or a - published engine port). + caller of no-docker.sh had (NO_DOCKER_ORIGINAL_HOST); every socket under + /run, /var/run, /tmp, $TMPDIR and ~/.docker; loopback ports 2375 and + 2376; and every listening TCP port. An endpoint is an engine if it accepts a connection and answers `GET /_ping` (sent with the User-Agent no-docker-probe) with an API version -header. The endpoints are only contacted once the mechanism is proven, so -outside the isolation nothing is sent to an engine. Prints one line per -check. +header. Nothing is written, connected to or sent to an engine before the +mechanism checks passed. Prints one line per check. + +Modes used by no-docker.sh outside the isolation: + --listening print the listening TCP port numbers (macOS), one per line; + --sockets print, NUL-separated, every Unix socket path on the local + file systems and every bound pathname socket of this + network namespace (Linux); fails when its time budget runs + out; + --seal WORK as root in the new mount namespace: make every mount + reachable by path read-only, except the private file systems + and WORK, and verify it (Linux). """ import concurrent.futures +import ctypes import errno import glob import json import os +import re import socket import stat import subprocess import sys import tempfile +import time ok = True @@ -93,33 +115,6 @@ def engine_answers(family, address, timeout=2): return None -def check_mechanism_darwin(): - d = tempfile.mkdtemp(prefix="no-docker-probe.") - path = os.path.join(d, "s") - srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - try: - srv.bind(path) - srv.listen(1) - report(denied(socket.AF_UNIX, path), "macOS sandbox denies Unix socket connects") - finally: - srv.close() - os.unlink(path) - os.rmdir(d) - report(denied(socket.AF_INET, ("127.0.0.1", 2375)), "macOS sandbox denies loopback port 2375") - report(denied(socket.AF_INET, ("192.0.2.1", 2375)), "macOS sandbox denies non-loopback TCP") - if not ok: - # Not sandboxed: write nothing and connect to nothing else. - return - held = os.environ.get("NO_DOCKER_HELD_PORTS", "").split() - reached = [p for p in held if p.isdigit() and connects(("127.0.0.1", int(p)))] - report(not reached, f"macOS sandbox denies the {len(held)} loopback ports listening before it started" - + (": reached " + " ".join(reached) if reached else "")) - home = os.path.realpath(os.path.expanduser("~")) - tmp = os.path.realpath(os.environ.get("TMPDIR") or "/tmp") - for where, d in (("under $HOME", home), ("outside $TMPDIR", os.path.dirname(tmp)), ("in /tmp", "/private/tmp")): - report(not_writable(d), f"macOS sandbox denies writes {where} ({d})") - - def connects(address): """True if a TCP connect succeeds; nothing is sent.""" s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -133,48 +128,33 @@ def connects(address): return True -def not_writable(directory): - """True if a new file cannot be created in directory.""" +def create_error(directory): + """None if a new file can be created in directory (it is removed again), + else the errno of the failure.""" path = os.path.join(directory, f".no-docker-probe-{os.getpid()}") try: - fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) except FileExistsError: - return False - except OSError: - return True + return None + except OSError as e: + return e.errno os.close(fd) os.unlink(path) - return False + return None -def check_mechanism_linux(): - nnp = "" - groups = "" - with open("/proc/self/status") as f: - for line in f: - if line.startswith("NoNewPrivs:"): - nnp = line.split()[1] - elif line.startswith("Groups:"): - groups = line[len("Groups:"):].strip() - report(nnp == "1", f"no_new_privs is set ({nnp or 'unknown'})") - report(groups == "", f"no supplementary groups ({groups or 'none'})") - names = sorted(n for _, n in socket.if_nameindex()) - report(names == ["lo"], f"network namespace has only loopback ({' '.join(names)})") - run_tmpfs = False - with open("/proc/self/mountinfo") as f: - for line in f: - fields = line.split() - sep = fields.index("-") - if fields[4] == "/run" and fields[sep + 1] == "tmpfs": - run_tmpfs = True # the last mount on /run is the visible one - elif fields[4] == "/run": - run_tmpfs = False - report(run_tmpfs, "/run is a private tmpfs") - sockets = [p for p in walk_sockets(["/run"])] - report(not sockets, f"no socket under /run ({' '.join(sockets) or 'none'})") +def not_writable(directory): + return create_error(directory) is not None + + +class WalkLimit(Exception): + pass def walk_sockets(roots, limit=20000): + """Socket paths under roots; raises WalkLimit instead of stopping + silently when more than limit entries would have to be looked at.""" + found = [] seen = 0 for root in roots: if not root or not os.path.isdir(root): @@ -183,22 +163,348 @@ def walk_sockets(roots, limit=20000): for name in dirnames + filenames: seen += 1 if seen > limit: - return + raise WalkLimit(f"more than {limit} entries under {' '.join(r for r in roots if r)}") p = os.path.join(dirpath, name) try: if stat.S_ISSOCK(os.lstat(p).st_mode): - yield p + found.append(p) except OSError: pass + return found + + +# --- macOS ------------------------------------------------------------------- + +def check_mechanism_darwin(): + d = tempfile.mkdtemp(prefix="no-docker-probe.") + # Bound and connected by a relative path: the absolute one can exceed + # the 104 bytes of sun_path. + cwd = os.open(".", os.O_RDONLY) + srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + os.chdir(d) + srv.bind("s") + srv.listen(1) + report(denied(socket.AF_UNIX, "s"), "macOS sandbox denies Unix socket connects") + finally: + srv.close() + try: + os.unlink("s") + except OSError: + pass + os.fchdir(cwd) + os.close(cwd) + os.rmdir(d) + report(denied(socket.AF_INET, ("127.0.0.1", 2375)), "macOS sandbox denies loopback port 2375") + report(denied(socket.AF_INET, ("192.0.2.1", 2375)), "macOS sandbox denies non-loopback TCP") + if not ok: + # Not sandboxed: write nothing, signal nothing and connect to nothing else. + return + held = os.environ.get("NO_DOCKER_HELD_PORTS", "").split() + reached = [p for p in held if p.isdigit() and connects(("127.0.0.1", int(p)))] + report(not reached, f"macOS sandbox denies the {len(held)} loopback ports listening before it started" + + (": reached " + " ".join(reached) if reached else "")) + outer = os.environ.get("NO_DOCKER_OUTER_PID", "") + result = "unknown" + if outer.isdigit(): + try: + os.kill(int(outer), 0) + result = "reached" + except PermissionError: + result = "EPERM" + except OSError as e: + result = errno.errorcode.get(e.errno, str(e.errno)) + report(result == "EPERM", f"macOS sandbox denies signals to processes outside it (no-docker.sh {outer or '?'}: {result})") + home = os.path.realpath(os.path.expanduser("~")) + user_tmp = os.environ.get("NO_DOCKER_USER_TMP", "") + report(bool(user_tmp), "the user's temporary directory is known (NO_DOCKER_USER_TMP)") + for where, d in (("under $HOME", home), ("in the user's temporary directory", user_tmp), + ("in /tmp", "/private/tmp"), ("in /private/var/tmp", "/private/var/tmp")): + if d: + report(not_writable(d), f"macOS sandbox denies writes {where} ({d})") + tmpdir = os.environ.get("TMPDIR", "") + report(bool(tmpdir) and create_error(tmpdir) is None, f"$TMPDIR of the run is writable ({tmpdir})") + + +# --- Linux ------------------------------------------------------------------- + +# File systems without files of their own that could be sockets of a +# service, or read-only images (squashfs snaps). +PSEUDO = {"proc", "sysfs", "cgroup", "cgroup2", "devpts", "mqueue", "debugfs", "tracefs", "securityfs", + "pstore", "bpf", "configfs", "fusectl", "hugetlbfs", "binfmt_misc", "autofs", "efivarfs", "nsfs", + "rpc_pipefs", "squashfs", "erofs", "iso9660"} +# Mounts that stay writable inside: the private file systems, /proc and /dev +# (device nodes; no file can be created there without root). +WRITABLE = ("/tmp", "/var/tmp", "/dev/shm", "/run", "/var/run", "/dev/mqueue", "/proc", "/dev", "/dev/pts") +PRIVATE = (("/tmp", "tmpfs"), ("/var/tmp", "tmpfs"), ("/dev/shm", "tmpfs"), ("/run", "tmpfs"), + ("/dev/mqueue", "mqueue")) +HOST_COMMS = {"systemd", "init", "Runner.Worker", "Runner.Listener", "dockerd", "containerd", "sshd", + "containerd-shim", "rootlesskit", "podman", "dbus-daemon"} + + +def unescape(s): + return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), s) + + +def mounts(): + entries = [] + with open("/proc/self/mountinfo") as f: + for line in f: + fields = line.split() + sep = fields.index("-") + entries.append({"id": int(fields[0]), "point": unescape(fields[4]), + "opts": fields[5].split(","), "fstype": fields[sep + 1]}) + return entries + + +def mount_id_of(path): + """The mount id of the file at path (not following a final symbolic + link), or None if path cannot be reached.""" + try: + fd = os.open(path, os.O_PATH | os.O_NOFOLLOW) + except OSError as e: + if e.errno in (errno.ENOENT, errno.EACCES, errno.ENOTDIR, errno.ELOOP): + return None + raise + try: + with open(f"/proc/self/fdinfo/{fd}") as f: + for line in f: + if line.startswith("mnt_id:"): + return int(line.split()[1]) + finally: + os.close(fd) + raise OSError(f"no mnt_id for {path}") + + +def visible_mounts(): + """The mounts reachable by path: a mount hidden under another one, or + under a path that no longer leads to it, cannot be written through.""" + return [m for m in mounts() if mount_id_of(m["point"]) == m["id"]] + + +def writable_mounts(work): + keep = set(WRITABLE) | {work} + return [m["point"] for m in visible_mounts() if m["point"] not in keep and "ro" not in m["opts"]] + + +MS_RDONLY, MS_NOSUID, MS_NODEV, MS_NOEXEC, MS_REMOUNT = 1, 2, 4, 8, 32 +MS_NOSYMFOLLOW, MS_NOATIME, MS_NODIRATIME, MS_BIND = 256, 1024, 2048, 4096 +MS_RELATIME, MS_STRICTATIME = 1 << 21, 1 << 24 +KEEP_FLAGS = {"nosuid": MS_NOSUID, "nodev": MS_NODEV, "noexec": MS_NOEXEC, "nosymfollow": MS_NOSYMFOLLOW, + "noatime": MS_NOATIME, "nodiratime": MS_NODIRATIME, "relatime": MS_RELATIME, + "strictatime": MS_STRICTATIME} + + +def seal(work): + """As root in the new mount namespace: remount every reachable mount + except WRITABLE and work read-only (a bind remount changes only this + namespace's copy and keeps its other per-mount flags), then verify.""" + libc = ctypes.CDLL(None, use_errno=True) + libc.mount.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulong, ctypes.c_void_p] + keep = set(WRITABLE) | {work} + count = 0 + for m in visible_mounts(): + if m["point"] in keep or "ro" in m["opts"]: + continue + flags = MS_REMOUNT | MS_BIND | MS_RDONLY + for o in m["opts"]: + flags |= KEEP_FLAGS.get(o, 0) + if libc.mount(None, os.fsencode(m["point"]), None, flags, None) != 0: + e = ctypes.get_errno() + print(f"no-docker: cannot make {m['point']} read-only: {os.strerror(e)}", file=sys.stderr) + return 1 + count += 1 + left = writable_mounts(work) + if left: + print(f"no-docker: still writable after the remount: {' '.join(left)}", file=sys.stderr) + return 1 + print(f"no-docker: {count} mounts made read-only", file=sys.stderr) + return 0 + + +def enumerate_sockets(budget=180): + """Every socket file on the local file systems (find -xdev per mount) + and every bound pathname socket of this network namespace + (/proc/net/unix, what ss -xl shows). Raises when the budget runs out.""" + start = time.monotonic() + found = set() + points = sorted({m["point"] for m in mounts() if m["fstype"] not in PSEUDO}) + for p in points: + left = budget - (time.monotonic() - start) + if left <= 0: + raise TimeoutError(f"the {budget}s budget ran out before {p}") + try: + r = subprocess.run(["find", p, "-xdev", "-type", "s", "-print0"], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, timeout=left) + except subprocess.TimeoutExpired: + raise TimeoutError(f"the {budget}s budget ran out in {p}") from None + if r.returncode not in (0, 1): # 1: some directories were unreadable + raise OSError(f"find {p} exited with {r.returncode}") + found.update(x for x in r.stdout.split(b"\0") if x) + with open("/proc/net/unix", "rb") as f: + next(f) + for line in f: + parts = line.split(None, 7) + if len(parts) == 8 and parts[7].startswith(b"/"): + found.add(parts[7].rstrip(b"\n")) + return sorted(found), len(points), time.monotonic() - start + + +def unix_connects(path): + """The socket type name a connect to path succeeds with, None if none.""" + for kind, name in ((socket.SOCK_STREAM, "stream"), (socket.SOCK_SEQPACKET, "seqpacket"), + (socket.SOCK_DGRAM, "dgram")): + s = socket.socket(socket.AF_UNIX, kind) + s.settimeout(2) + try: + s.connect(path) + return name + except socket.timeout: + return name + " (timeout)" # a listener with a full backlog + except OSError as e: + if e.errno == errno.EPROTOTYPE: + continue + return None + finally: + s.close() + return None + + +def status(pid="self"): + st = {} + with open(f"/proc/{pid}/status") as f: + for line in f: + k, _, v = line.partition(":") + st[k] = v.strip() + return st + + +def env_pairs(name): + return dict(x.split("=", 1) for x in os.environ.get(name, "").split() if "=" in x) + + +def check_mechanism_linux(): + me = os.getuid() + st = status() + uids = st.get("Uid", "0").split() + report("0" not in uids, f"not root (uids {' '.join(uids)})") + caps = {k: st.get(k, "unknown") for k in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb")} + report(all(v != "unknown" and int(v, 16) == 0 for v in caps.values()), + "no capabilities (" + " ".join(f"{k}={v}" for k, v in caps.items()) + ")") + report(st.get("NoNewPrivs") == "1", f"no_new_privs is set ({st.get('NoNewPrivs', 'unknown')})") + report(st.get("Groups", "x") == "", f"no supplementary groups ({st.get('Groups') or 'none'})") + + host_ns = env_pairs("NO_DOCKER_HOST_NS") + for ns in ("net", "mnt", "pid", "ipc"): + own = os.readlink(f"/proc/self/ns/{ns}") + host = host_ns.get(ns, "") + report(bool(host) and own != host, f"private {ns} namespace ({own}; caller {host or 'unknown'})") + nspid = st.get("NSpid", "").split() + report(len(nspid) >= 2, f"in a pid namespace below the caller's (NSpid {' '.join(nspid)})") + report(os.readlink("/proc/self") == str(os.getpid()), "/proc belongs to this pid namespace") + foreign = [] + count = 0 + for d in os.listdir("/proc"): + if not d.isdigit(): + continue + try: + pst = status(d) + except OSError: + continue # ended meanwhile + count += 1 + name, puids = pst.get("Name", "?"), pst.get("Uid", "").split() + if d == "1": + if puids[:1] != ["0"] or name not in ("sh", "dash"): + foreign.append(f"1:{name}:{'/'.join(puids)}") + elif any(u != str(me) for u in puids) or name in HOST_COMMS: + foreign.append(f"{d}:{name}:{'/'.join(puids)}") + report(not foreign, f"only our processes and the root pid 1 of the isolation are visible ({count} processes" + + (f"; foreign: {' '.join(foreign)}" if foreign else "") + ")") + sid = os.getsid(0) + sid_ours = False + if sid > 1: + try: + sid_ours = all(u == str(me) for u in status(sid).get("Uid", "").split()) + except OSError: + pass + report(sid_ours, f"a session of our own (session {sid})") + try: + os.close(os.open("/dev/tty", os.O_RDWR)) + report(False, "no controlling terminal (TIOCSTI)") + except OSError as e: + report(True, f"no controlling terminal ({errno.errorcode.get(e.errno, e.errno)})") + + names = sorted(n for _, n in socket.if_nameindex()) + report(names == ["lo"], f"network namespace has only loopback ({' '.join(names)})") + + work = os.environ.get("NO_DOCKER_RUN_DIR", "") + report(bool(work), f"the run's directory is known ({work or 'NO_DOCKER_RUN_DIR unset'})") + host_devs = env_pairs("NO_DOCKER_HOST_DEVS") + vis = {m["point"]: m for m in visible_mounts()} + for path, fstype in PRIVATE: + m = vis.get(path) + try: + dev = str(os.stat(path).st_dev) + except OSError: + dev = "?" + host = host_devs.get(path, "") + report(bool(m) and m["fstype"] == fstype and bool(host) and dev != host, + f"{path} is a private {fstype} (device {dev}; caller {host or 'unknown'})") + m = vis.get("/sys/fs/cgroup") + report(bool(m) and m["fstype"] == "tmpfs" and "ro" in m["opts"], "/sys/fs/cgroup is masked by a read-only tmpfs") + if work: + left = writable_mounts(work) + report(not left, f"every other mount is read-only ({len(vis)} reachable mounts" + + (f"; writable: {' '.join(left)}" if left else "") + ")") + try: + report(mount_id_of(".") == mount_id_of(os.getcwd()), f"the working directory is reachable by its path ({os.getcwd()})") + except OSError as e: + report(False, f"the working directory is reachable by its path ({e})") + if not ok: + # Not isolated: write nothing and connect to nothing. + return + + protected = [p for p in os.environ.get("NO_DOCKER_PROTECTED", "").split(":") if p] + report(bool(protected), "the protected directories are known (NO_DOCKER_PROTECTED)") + for p in protected: + e = create_error(p) + report(e in (errno.EROFS, errno.EACCES), f"writes to {p} fail ({errno.errorcode.get(e, e) if e else 'written'})") + writable = [] + try: + for dirpath, dirnames, filenames in os.walk("/sys/fs/cgroup"): + for n in dirnames + filenames: + if os.access(os.path.join(dirpath, n), os.W_OK): + writable.append(os.path.join(dirpath, n)) + writable += [p for p in ("/sys/fs/cgroup/cgroup.kill", "/sys/fs/cgroup/cgroup.procs") if os.access(p, os.W_OK)] + except OSError as e: + writable.append(str(e)) + report(not writable, f"no cgroup file is writable ({' '.join(writable) or 'none'})") + try: + found = walk_sockets(["/run", "/tmp", "/var/tmp", "/dev/shm"], limit=20000) + report(not found, f"no socket in the private directories ({' '.join(found) or 'none'})") + except WalkLimit as e: + report(False, f"no socket in the private directories ({e})") + listed = os.environ.get("NO_DOCKER_SOCKETS", "") + try: + with open(listed, "rb") as f: + paths = [os.fsdecode(p) for p in f.read().split(b"\0") if p] + except OSError as e: + report(False, f"the Unix sockets listed outside can be read ({listed or 'NO_DOCKER_SOCKETS unset'}: {e})") + return + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: + reached = [f"{p} ({k})" for p, k in zip(paths, pool.map(unix_connects, paths)) if k] + report(not reached, f"none of the {len(paths)} Unix sockets listed outside accepts a connection" + + (": " + ", ".join(reached) if reached else "")) def listening_ports(strict=False): - """(host, port) of TCP ports listening on loopback or on all addresses. + """(host, port) of listening TCP ports. strict (macOS): raise instead of returning a list that may be incomplete. The list is accepted, even empty, only if netstat succeeded, printed its column header and every LISTEN row has a parsable local - address.""" + address. Every address counts (loopback, all, link-local or another): + no-docker.sh holds the port number.""" ports = set() if sys.platform == "darwin": try: @@ -222,8 +528,7 @@ def listening_ports(strict=False): if strict: raise OSError(f"unparsable netstat LISTEN row: {line.strip()}") continue - if host in ("127.0.0.1", "::1", "*"): - ports.add(("::1" if host == "::1" else "127.0.0.1", int(port))) + ports.add(("::1" if ":" in host else "127.0.0.1", int(port))) else: for name in ("/proc/net/tcp", "/proc/net/tcp6"): try: @@ -275,7 +580,14 @@ def endpoints(): tcp.add((host.strip("[]"), int(port))) roots = ["/run", "/var/run", "/tmp", os.environ.get("TMPDIR", ""), f"{home}/.docker", f"{home}/Library/Containers/com.docker.docker/Data"] - unix.update(walk_sockets(roots)) + try: + unix.update(walk_sockets(roots)) + except WalkLimit as e: + if sys.platform != "darwin": + raise + # The sandbox denies every Unix socket connect (checked first); the + # walk is a second line only. Said, not hidden. + print(f"no-docker: info socket walk incomplete, {e}; Unix sockets are denied by the sandbox") return sorted(unix), sorted(tcp) @@ -289,16 +601,42 @@ def main(): return 1 print("\n".join(sorted({str(port) for _, port in ports}, key=int))) return 0 - if sys.platform == "darwin": - check_mechanism_darwin() - elif sys.platform.startswith("linux"): - check_mechanism_linux() - else: - report(False, f"no isolation mechanism for platform {sys.platform}") + if sys.argv[1:] == ["--sockets"]: + try: + paths, points, took = enumerate_sockets() + except (OSError, TimeoutError, StopIteration) as e: + print(f"no-docker: cannot list the Unix sockets: {e}", file=sys.stderr) + return 1 + sys.stdout.buffer.write(b"".join(p + b"\0" for p in paths)) + print(f"no-docker: {len(paths)} Unix sockets listed on {points} mounts in {took:.1f}s", file=sys.stderr) + return 0 + if sys.argv[1:2] == ["--seal"] and len(sys.argv) == 3: + try: + return seal(sys.argv[2]) + except OSError as e: + print(f"no-docker: cannot make the mounts read-only: {e}", file=sys.stderr) + return 1 + if sys.argv[1:]: + print(f"no-docker: unknown arguments {' '.join(sys.argv[1:])}", file=sys.stderr) + return 2 + try: + if sys.platform == "darwin": + check_mechanism_darwin() + elif sys.platform.startswith("linux"): + check_mechanism_linux() + else: + report(False, f"no isolation mechanism for platform {sys.platform}") + except OSError as e: + report(False, f"the isolation check failed: {e}") if not ok: # Not isolated: do not send anything to an engine that may answer. return 1 - unix, tcp = endpoints() + try: + unix, tcp = endpoints() + except WalkLimit as e: + report(False, f"every socket of the walked directories was checked ({e})") + return 1 + def ask(target): kind, addr = target try: diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh index 9965bbc..f2f7381 100755 --- a/scripts/ci/no-docker.sh +++ b/scripts/ci/no-docker.sh @@ -1,46 +1,51 @@ #!/usr/bin/env bash # -# Run a command where no container engine can be reached, so a test suite, -# and any modified or mutated copy of the code it runs, cannot touch the -# Docker daemon of the machine (its images, containers, volumes, networks -# or build cache) by the routes below: docker found on PATH or by an -# absolute path, in a nested shell or a command substitution, with -H, -# --context or an inherited DOCKER_HOST, curl --unix-socket or a raw socket. +# Run a command where no container engine can be reached and where it cannot +# change what runs or is read outside it later, so a test suite, and any +# modified or mutated copy of the code it runs, cannot touch the Docker +# daemon of the machine (its images, containers, volumes, networks or build +# cache) or the user's environment. docs/dev-environment.md, "Isolation from +# Docker", lists every property, how it is verified and what is left. # -# Usage: no-docker.sh COMMAND [ARG...] run COMMAND isolated; its status -# no-docker.sh --check exit 0 only if this process is -# already isolated +# Usage: no-docker.sh [--work DIR] COMMAND [ARG...] +# run COMMAND isolated; its status +# no-docker.sh --check exit 0 only if this process is already +# isolated # -# Isolation: -# macOS sandbox-exec: every connect to a Unix socket is denied, as is TCP -# to anything but loopback, to loopback ports 22, 2375 and 2376 -# (ssh and the Docker API ports) and to every loopback port that -# was listening before the command started (local services: a -# development server, an IDE, a published container port or a -# socket proxy); -# the list of those ports must be complete, or the command is -# refused; nothing can be written except under $TMPDIR, /dev and -# bash 3.2's here-document files in /tmp (not $HOME, where Docker -# Desktop keeps its disk image, nor Homebrew's prefix, -# /Applications or other tools' files in /tmp, which run or are -# read outside the sandbox later), so $TMPDIR must not be /tmp or -# under it or under $HOME; no preference can be written through -# cfprefsd and no keystroke injected into the terminal (TIOCSTI); -# open, osascript, launchctl and ssh cannot run, and jobs cannot be -# submitted to launchd, nor Launch Services or Apple Events used -# (the known ways to have a program started outside the sandbox). -# The profile filters these routes; it is not a complete jail. -# Setuid programs cannot run in the sandbox, so ps is an -# unprivileged, ad-hoc signed copy. Loopback ports the command -# opens itself stay reachable for the tests; the probe asks every -# listening port whether it is an engine. -# Linux sudo unshare: a network namespace with only loopback and a mount -# namespace with an empty tmpfs over /run (and /var/run), then the -# calling user without supplementary groups (no docker group) and -# with no_new_privs, so sudo cannot regain root to undo it. -# In both, DOCKER_* and similar variables are removed and DOCKER_CONFIG is -# an empty directory. Before COMMAND runs, no-docker-probe.py proves from -# inside that the mechanism is in effect and that no engine answers on any +# --work DIR: use DIR, a fresh, empty directory of the caller (mode 0700, not +# a symbolic link and without one in its path), as the only writable +# directory of the command instead of a temporary one, and keep it +# afterwards. On macOS DIR must be under $TMPDIR or /private/var/tmp, on +# Linux under /var/tmp. Whatever the command leaves there is untrusted data: +# a later reader must not follow symbolic links, must accept regular files +# only, must limit their size and must not run them. +# +# Supported: macOS developer machines (sandbox-exec) and Linux on +# GitHub-hosted Actions runners (sudo unshare). Everything else, including +# Linux elsewhere and uid 0, is refused. +# macOS sandbox-exec: no connect to a Unix socket, no TCP beyond +# loopback, to loopback ports 22, 2375 and 2376 or to a loopback +# port that was listening before the command started; file writes +# only in the run's own directory (the command's $TMPDIR is in it), +# the /dev nodes a shell needs and bash 3.2's here-document files in +# /tmp; no signal to a process outside the sandbox; no preference +# write, TIOCSTI, open, osascript, launchctl, ssh, launchd job, Launch +# Services, Apple Events, login item or background task registration +# and no lookup of Docker's own services. $TMPDIR must be the +# per-user directory under /private/var/folders. +# Linux sudo unshare: new network (loopback only), mount, pid, and IPC +# namespaces; empty tmpfs over /run, /var/run, /tmp, /var/tmp, +# /dev/shm and /sys/fs/cgroup, a fresh /dev/mqueue, every other mount +# read-only; the command runs as the calling user without +# supplementary groups, capabilities or the means to regain them +# (no_new_privs), in a session of its own, with an allowlisted +# environment, under a root pid 1 that waits for it; when it ends +# or the isolation is killed, every process left inside is killed. +# Unix sockets found outside before the start must not accept a +# connection inside. +# In both, the variables that select an engine are removed and DOCKER_CONFIG +# is an empty directory. Before COMMAND runs, no-docker-probe.py proves from +# inside that each mechanism is in effect and that no engine answers on any # endpoint it can find. If isolation cannot be applied or proven, COMMAND # does not run and the exit status is 125; so is the status if the # isolation tool returns without the command having started inside it. @@ -69,63 +74,202 @@ if [ "${1-}" = --inner ]; then exec "$@" fi +given_work="" +if [ "${1-}" = --work ]; then + [ $# -ge 2 ] || refuse "--work needs a directory" + given_work=$2 + shift 2 +fi [ $# -gt 0 ] || refuse "no command given" if [ -n "${NO_DOCKER_ACTIVE-}" ]; then # Already isolated (a suite re-executing itself): never nest. python3 -I "$probe" >/dev/null || refuse "inside no-docker.sh, but the isolation check fails" + [ -z "$given_work" ] || refuse "--work cannot be used inside the isolation" exec "$@" fi -work=$(mktemp -d "${TMPDIR:-/tmp}/no-docker.XXXXXX") || refuse "no temporary directory" -work=$(cd -P "$work" && pwd -P) -trap 'rm -rf "$work"' EXIT -mkdir "$work/docker-config" "$work/bin" || refuse "cannot prepare $work" +# uid 0 keeps every capability through the isolation; refused first. +if [ "$(id -u)" = 0 ] || [ "$(id -ru)" = 0 ] || [ "${EUID-}" = 0 ] || [ "${UID-}" = 0 ]; then + refuse "running as root (uid 0); run it as an unprivileged user" +fi + +os=$(uname -s) +uid=$(id -u) + +# plain PATH: true if PATH has only characters that are safe in a sandbox +# profile and in the shell of the namespace. +plain() { + case $1 in "" | *[!A-Za-z0-9/._-]*) return 1 ;; esac + return 0 +} + +# owner_mode PATH: " " of PATH. +owner_mode() { + if [ "$os" = Darwin ]; then stat -f '%u %Lp' "$1"; else stat -c '%u %a' "$1"; fi +} + +# use_work DIR PARENT...: check the caller's --work DIR and use it as $work. +use_work() { + local d=$1 canon real p inside=no + shift + case $d in /*) ;; *) refuse "--work $d is not an absolute path" ;; esac + plain "$d" || refuse "unexpected character in --work $d" + case $d in */ | *//* | */./* | */../* | */. | */..) refuse "--work $d is not a canonical path" ;; esac + [ -d "$d" ] && [ ! -L "$d" ] || refuse "--work $d is not a directory or is a symbolic link" + canon=$d + if [ "$os" = Darwin ]; then + case $d in /var/* | /tmp/*) canon=/private$d ;; esac + fi + real=$(cd -P "$d" && pwd -P) || refuse "cannot resolve --work $d" + [ "$real" = "$canon" ] || refuse "--work $d has a symbolic link in its path ($real)" + for p in "$@"; do + case $real in "$p"/?*) inside=yes ;; esac + done + [ "$inside" = yes ] || refuse "--work $d is not under $*" + case $real/ in "$home"/*) refuse "--work $d is under \$HOME" ;; esac + case $home/ in "$real"/*) refuse "--work $d contains \$HOME" ;; esac + [ "$(owner_mode "$real")" = "$uid 700" ] || refuse "--work $d is not owned by you with mode 0700" + [ -z "$(ls -A "$real")" ] || refuse "--work $d is not empty" + work=$real +} + +home=$(cd -P "$HOME" && pwd -P) || refuse "cannot resolve \$HOME" +plain "$home" || refuse "unexpected character in \$HOME ($home)" -# The environment of the command: the caller's, without the variables that -# select a container engine or its configuration. +case $os in + Darwin) + command -v sandbox-exec >/dev/null || refuse "sandbox-exec not found" + # $TMPDIR must be the per-user temporary directory macOS creates, + # /private/var/folders///T, owned by the user with mode 0700: + # any other directory may hold files that run or are read outside the + # sandbox. The command gets a directory of its own inside it. + t=${TMPDIR-} + [ -n "$t" ] || t=$(getconf DARWIN_USER_TEMP_DIR 2>/dev/null) || refuse "TMPDIR is unset and getconf DARWIN_USER_TEMP_DIR fails" + tmp=$(cd -P "$t" 2>/dev/null && pwd -P) || refuse "cannot resolve the temporary directory $t" + plain "$tmp" || refuse "unexpected character in the temporary directory $tmp" + case $tmp in + /private/var/folders/[!/][!/]/?*/T) ;; + *) refuse "the temporary directory $tmp is not the per-user directory /private/var/folders///T; unset TMPDIR or set it to that directory" ;; + esac + rest=${tmp#/private/var/folders/??/} + case ${rest%/T} in "" | */*) refuse "the temporary directory $tmp is not the per-user directory /private/var/folders///T" ;; esac + [ "$(owner_mode "$tmp")" = "$uid 700" ] || refuse "the temporary directory $tmp is not owned by you with mode 0700" + if [ -n "$given_work" ]; then + use_work "$given_work" "$tmp" /private/var/tmp + else + work=$(mktemp -d "$tmp/no-docker.XXXXXX") || refuse "no temporary directory" + work=$(cd -P "$work" && pwd -P) || refuse "cannot resolve the temporary directory" + trap 'rm -rf "$work"' EXIT + fi + ;; + Linux) + # Only GitHub-hosted runners: the VM is discarded after the job, sudo + # needs no password and no rootless engine or user session exists. The + # checks below guard against using this elsewhere by accident; they are + # not a boundary against the caller. + [ "${GITHUB_ACTIONS-}" = true ] && [ "${RUNNER_ENVIRONMENT-}" = github-hosted ] || + refuse "Linux is supported on GitHub-hosted CI runners only (GITHUB_ACTIONS and RUNNER_ENVIRONMENT)" + found=no + pid=$PPID + n=0 + while [ "$pid" -gt 1 ] && [ "$n" -lt 64 ]; do + comm=$(cat "/proc/$pid/comm" 2>/dev/null) || break + if [ "$comm" = Runner.Worker ]; then + exe=$(readlink "/proc/$pid/exe" 2>/dev/null) || break + [ "${exe##*/}" = Runner.Worker ] && found=yes + break + fi + st=$(cat "/proc/$pid/stat" 2>/dev/null) || break + read -r _ pid _ <<<"${st##*) }" + case $pid in "" | *[!0-9]*) break ;; esac + n=$((n + 1)) + done + [ "$found" = yes ] || refuse "Linux is supported on GitHub-hosted CI runners only (no Runner.Worker ancestor)" + protected=$home + for v in GITHUB_WORKSPACE RUNNER_TEMP; do + d=$(cd -P "${!v:-/nonexistent}" 2>/dev/null && pwd -P) || refuse "Linux is supported on GitHub-hosted CI runners only ($v is not a directory)" + plain "$d" || refuse "unexpected character in $v ($d)" + case $d in "$home"/?*) ;; *) refuse "Linux is supported on GitHub-hosted CI runners only ($v is not under \$HOME)" ;; esac + protected="$protected:$d" + done + for tool in sudo unshare setpriv setsid ip mount find stat python3; do + command -v "$tool" >/dev/null || refuse "$tool not found" + done + sudo -n true 2>/dev/null || refuse "creating the namespaces needs passwordless sudo" + if [ -n "$given_work" ]; then + use_work "$given_work" /var/tmp + else + work=$(mktemp -d /tmp/no-docker.XXXXXX) || refuse "no temporary directory" + work=$(cd -P "$work" && pwd -P) || refuse "cannot resolve the temporary directory" + trap 'rm -rf "$work"' EXIT + fi + cwd=$(pwd -P) || refuse "cannot resolve the working directory" + # What the probe compares inside: the namespaces and the devices of the + # directories that must be private. + host_ns="" + for ns in net mnt pid ipc; do + v=$(readlink "/proc/self/ns/$ns") || refuse "cannot read the $ns namespace" + host_ns="$host_ns $ns=$v" + done + host_devs="" + for d in /tmp /var/tmp /dev/shm /run /dev/mqueue; do + v=$(stat -c %d "$d") || refuse "cannot stat $d" + host_devs="$host_devs $d=$v" + done + ;; + *) + refuse "no isolation mechanism for $os" + ;; +esac +plain "$work" || refuse "unexpected character in the working directory $work" +mkdir "$work/docker-config" "$work/bin" "$work/tmp" || refuse "cannot prepare $work" + +# The environment of the command. envargs=() while IFS= read -r -d '' kv; do - case ${kv%%=*} in - DOCKER_* | CONTAINER_HOST | CONTAINER_CONNECTION | CONTAINERD_* | BUILDKIT_HOST | COMPOSE_* | PODMAN_* | NO_DOCKER_*) ;; - *) envargs+=("$kv") ;; - esac + name=${kv%%=*} + if [ "$os" = Linux ]; then + # An allowlist: no session bus, display, agent, runner file-command or + # token variables. + case $name in + PATH | HOME | USER | LOGNAME | SHELL | LANG | LANGUAGE | LC_* | TERM | TZ | CI | DEVENV | DEVENV_* | GUARD | GUARD_* | ADAPTED | HARNESS | FLOWCRM_*) envargs+=("$kv") ;; + esac + else + case $name in + DOCKER_* | CONTAINER_HOST | CONTAINER_CONNECTION | CONTAINERD_* | BUILDKIT_HOST | COMPOSE_* | PODMAN_* | NO_DOCKER_* | TMPDIR) ;; + *) envargs+=("$kv") ;; + esac + fi done < <(env -0) -envargs+=("DOCKER_CONFIG=$work/docker-config" "NO_DOCKER_ACTIVE=1" "NO_DOCKER_ORIGINAL_HOST=${DOCKER_HOST-}" - "NO_DOCKER_STARTED=$work/started") +envargs+=("DOCKER_CONFIG=$work/docker-config" "TMPDIR=$work/tmp" "NO_DOCKER_ACTIVE=1" + "NO_DOCKER_ORIGINAL_HOST=${DOCKER_HOST-}" "NO_DOCKER_STARTED=$work/started" "NO_DOCKER_RUN_DIR=$work") -case $(uname -s) in +case $os in Darwin) - command -v sandbox-exec >/dev/null || refuse "sandbox-exec not found" # A copy of ps loses its setuid bit; it needs a signature of its own. { cp /bin/ps "$work/bin/ps" && codesign -f -s - "$work/bin/ps" 2>/dev/null; } || refuse "cannot prepare an unprivileged ps" for i in "${!envargs[@]}"; do case ${envargs[$i]} in PATH=*) envargs[i]="PATH=$work/bin:${envargs[$i]#PATH=}" ;; esac done - home=$(cd -P "$HOME" && pwd -P) || refuse "cannot resolve \$HOME" - tmp=$(cd -P "${TMPDIR:-/tmp}" && pwd -P) || refuse "cannot resolve the temporary directory" - # The command may write only under $TMPDIR and /dev, and in /tmp only - # the here-document files of bash 3.2 (sh-thd*, which it creates there - # whatever $TMPDIR is): programs outside the sandbox run files from - # $HOME, Homebrew's prefix, /Applications or other programs' temporary - # files in /tmp. So $TMPDIR must be a directory of its own: not /, not - # under $HOME and not /tmp or under it. - case $tmp/ in - / | "$home"/* | /private/tmp/* | /tmp/*) - refuse "the temporary directory $tmp is /, /tmp or under /tmp or \$HOME; set TMPDIR to a private directory (the macOS default under /var/folders)" ;; - esac - case $tmp in *[!A-Za-z0-9/._-]*) refuse "unexpected character in the temporary directory $tmp" ;; esac # Local services that were listening before the command started (a # development server, an IDE, a published container port or a socket # proxy) are not the command's: it may connect only to loopback ports it - # opens itself. + # opens itself. Every listening port number is held, whatever address + # it listens on. listening=$(python3 -I "$probe" --listening) || refuse "cannot list the listening TCP ports" held="" for port in $listening; do case $port in *[!0-9]* | "") refuse "unexpected port $port" ;; esac held="$held (remote ip \"localhost:$port\")" done - envargs+=("NO_DOCKER_HELD_PORTS=$(printf '%s' "$listening" | tr '\n' ' ')") + envargs+=("NO_DOCKER_HELD_PORTS=$(printf '%s' "$listening" | tr '\n' ' ')" "NO_DOCKER_OUTER_PID=$$" + "NO_DOCKER_USER_TMP=$tmp") + # File writes: the run's directory; of /dev only what shells and + # interpreters open for writing (measured, docs/dev-environment.md); + # bash 3.2's here-document files, which it creates in /tmp whatever + # $TMPDIR is. Signals only within this sandbox. An SBPL error makes + # sandbox-exec fail, so the command never starts (125). profile="(version 1) (allow default) (deny network-outbound (remote unix-socket)) @@ -133,38 +277,73 @@ case $(uname -s) in (allow network-outbound (remote ip \"localhost:*\")) (deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")$held) (deny file-write*) -(allow file-write* (subpath \"$tmp\") (subpath \"/dev\") (literal \"/private/tmp\") (regex #\"^/private/tmp/sh-thd[^/]*\$\")) -(deny file-write* (subpath \"$home\")) +(allow file-write* (subpath \"$work\") (literal \"/dev/null\") (literal \"/dev/zero\") (literal \"/dev/tty\") (literal \"/dev/dtracehelper\") (regex #\"^/dev/fd/[0-9]+\$\") (literal \"/private/tmp\") (regex #\"^/private/tmp/sh-thd[^/]*\$\")) (deny user-preference-write) (deny file-ioctl (ioctl-command #x80017472)) (deny process-exec (literal \"/usr/bin/open\") (literal \"/usr/bin/osascript\") (literal \"/bin/launchctl\") (literal \"/usr/bin/ssh\")) (deny job-creation) (deny lsopen) (deny appleevent-send) -(deny mach-lookup (global-name \"com.apple.coreservices.launchservicesd\") (global-name \"com.apple.coreservices.appleevents\") (global-name-regex #\"^com\\.apple\\.lsd\\.\"))" +(deny mach-lookup (global-name \"com.apple.coreservices.launchservicesd\") (global-name \"com.apple.coreservices.appleevents\") (global-name-regex #\"^com\\.apple\\.lsd\\.\") (global-name \"com.apple.xpc.smd\") (global-name-regex #\"^com\\.apple\\.backgroundtaskmanagement\") (global-name-regex #\"^com\\.apple\\.coreservices\\.sharedfilelistd\") (global-name-regex #\"^com\\.docker\\.\") (global-name-regex #\"^com\\.electron\\.dockerdesktop\\.\")) +(deny signal) +(allow signal (target same-sandbox))" sandbox-exec -p "$profile" /usr/bin/env -i "${envargs[@]}" "$BASH" "$here/no-docker.sh" --inner "$@" + rc=$? ;; Linux) - for tool in unshare setpriv ip; do - command -v "$tool" >/dev/null || refuse "$tool not found" - done - sudo -n true 2>/dev/null || refuse "creating the namespaces needs passwordless sudo" - # shellcheck disable=SC2016 # expanded by the inner shell - sudo -n unshare --net --mount --propagation private -- /bin/sh -c ' - ip link set lo up || exit 125 - mount -t tmpfs -o mode=0755,nosuid,nodev,noexec no-docker /run || exit 125 + # Every Unix socket on the local file systems and every bound one of + # the host's network namespace; inside, none may accept a connection. + # A time budget that runs out refuses rather than truncating the list. + python3 -I "$probe" --sockets >"$work/sockets.list" || refuse "cannot list the Unix sockets" + envargs+=("NO_DOCKER_SOCKETS=/run/no-docker/sockets.list" "NO_DOCKER_HOST_NS=${host_ns# }" + "NO_DOCKER_HOST_DEVS=${host_devs# }" "NO_DOCKER_PROTECTED=$protected") + # pid 1 of the new pid namespace is this root shell. It does not exec: + # it keeps the parent-death signal of --kill-child (a credential change + # would clear it), reaps orphans while it waits, and its exit kills + # everything left in the namespace. + # shellcheck disable=SC2016 # expanded by the shell in the namespace + init=' + set -u + PATH=/usr/sbin:/usr/bin:/sbin:/bin + export PATH + die() { printf "no-docker: refused: %s; the command was not run\n" "$1" >&2; exit 125; } + uid=$1 gid=$2 work=$3 probe=$4 cwd=$5 + shift 5 + cd / || die "cannot leave the working directory" + ip link set lo up || die "cannot bring up loopback" + mount -t tmpfs -o mode=0755,nosuid,nodev,noexec no-docker /run || die "cannot hide /run" + mkdir /run/no-docker /run/no-docker/work || die "cannot prepare /run" + mount --bind "$work" /run/no-docker/work || die "cannot keep $work" + cp "$work/sockets.list" /run/no-docker/sockets.list && chmod 0444 /run/no-docker/sockets.list && + rm -f "$work/sockets.list" || die "cannot keep the socket list" if [ -d /var/run ] && [ ! -L /var/run ]; then - mount -t tmpfs -o mode=0755,nosuid,nodev,noexec no-docker /var/run || exit 125 + mount -t tmpfs -o mode=0755,nosuid,nodev,noexec no-docker /var/run || die "cannot hide /var/run" fi - uid=$1 gid=$2 - shift 2 - exec setpriv --reuid "$uid" --regid "$gid" --clear-groups --no-new-privs -- /usr/bin/env -i "$@" - ' no-docker "$(id -u)" "$(id -g)" "${envargs[@]}" "$BASH" "$here/no-docker.sh" --inner "$@" - ;; - *) - refuse "no isolation mechanism for $(uname -s)" + for d in /tmp /var/tmp /dev/shm; do + mount -t tmpfs -o mode=1777,nosuid,nodev no-docker "$d" || die "cannot hide $d" + done + mount -t mqueue -o nosuid,nodev,noexec no-docker /dev/mqueue || die "cannot mount a private /dev/mqueue" + mount -t tmpfs -o ro,mode=0555,nosuid,nodev,noexec no-docker /sys/fs/cgroup || die "cannot mask /sys/fs/cgroup" + mkdir -p "$work" && mount --move /run/no-docker/work "$work" || die "cannot move $work into the private temporary directory" + python3 -I "$probe" --seal "$work" || die "cannot make the other mounts read-only" + cd "$cwd" || die "the working directory $cwd is not visible in the isolation" + setpriv --reuid "$uid" --regid "$gid" --clear-groups --no-new-privs --inh-caps=-all --bounding-set=-all -- \ + setsid --wait /usr/bin/env -i "$@" & + child=$! + wait "$child" + exit $? + ' + # Workflow commands in the command's output are ignored: the token is + # random and never passed inside. + token=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') + [ "${#token}" -eq 32 ] || refuse "cannot make a random token" + printf '::stop-commands::%s\n' "$token" + sudo -n unshare --net --mount --pid --fork --kill-child --mount-proc --ipc --propagation private -- \ + /bin/sh -c "$init" no-docker-init "$uid" "$(id -g)" "$work" "$probe" "$cwd" \ + "${envargs[@]}" "$BASH" "$here/no-docker.sh" --inner "$@" + rc=$? + printf '::%s::\n' "$token" ;; esac -rc=$? [ -e "$work/started" ] || refuse "the isolation tool returned $rc without starting the command" exit "$rc" diff --git a/tests/devenv/unit.sh b/tests/devenv/unit.sh index 04d238e..cdd6182 100755 --- a/tests/devenv/unit.sh +++ b/tests/devenv/unit.sh @@ -21,8 +21,9 @@ set -u here=$(cd -P "$(dirname "$0")" && pwd -P) # The suite sources and runs bin/devenv, or a modified or mutated copy of # it, so it runs only where no container engine can be reached. +# NO_DOCKER_WORK= is the isolation's only writable directory (--work). "$here/../../scripts/ci/no-docker.sh" --check >/dev/null 2>&1 || - exec "$here/../../scripts/ci/no-docker.sh" "$BASH" "$here/unit.sh" "$@" + exec "$here/../../scripts/ci/no-docker.sh" ${NO_DOCKER_WORK:+--work "$NO_DOCKER_WORK"} "$BASH" "$here/unit.sh" "$@" # DEVENV= runs the suite against another copy (mutation checks). devenv_script=${DEVENV:-"$here/../../bin/devenv"} # shellcheck source=../../bin/devenv From 70168209b73cb7326ed79ffe9c4f740be037e3d3 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 09:02:10 +0100 Subject: [PATCH 11/77] test(ci): cover the isolation's writes, signals, namespaces, platforms and CI order T-11 to T-15 (macOS): writes outside the run directory, the TMPDIR allowlist, signals to an outside canary, integrity of the helpers run outside the isolation (kept read-only to the code under test and checked by sha256 before every bare run and every mutant), and --work. T-16 to T-18 (Linux): a canary inside checks the pid view, sockets, read-only mounts, private /tmp, masked cgroups and IPC; no daemon survives the command or a killed unshare; uid 0 and non-GitHub-hosted use are refused. T-19 checks statically that isolated steps come last in each CI job. The mutant's log moves into the run directory, the only place it may write; the payload text and its hash are unchanged. --- scripts/ci/tests/ci-isolation-order.py | 226 ++++++++ scripts/ci/tests/no-docker.test.sh | 686 +++++++++++++++++++++---- 2 files changed, 799 insertions(+), 113 deletions(-) create mode 100644 scripts/ci/tests/ci-isolation-order.py diff --git a/scripts/ci/tests/ci-isolation-order.py b/scripts/ci/tests/ci-isolation-order.py new file mode 100644 index 0000000..0daa180 --- /dev/null +++ b/scripts/ci/tests/ci-isolation-order.py @@ -0,0 +1,226 @@ +"""Check that isolated steps come last in every CI job (static). + +Usage: ci-isolation-order.py REPO_ROOT [WORKFLOW] (default .github/workflows/ci.yml) + +A step is isolated when a script it runs (named in its `run:` text, or in +the recipe of a `make` target it runs) calls scripts/ci/no-docker.sh: it +runs code under test, which may leave files or processes for later steps. +Code under test cannot write anything on the Linux runner's file systems +except private temporary directories, but the order is the second line: +after the first isolated step of a job, only isolated steps may follow. + +Also checked: + - the required jobs "Repository guard" and "Dev environment" exist and + each has an isolated step; + - no job or step has an `if:` condition (one could run a non-isolated + step after a failed isolated one, or skip a required check); + - every tracked file that mentions no-docker.sh is either part of the + isolation itself, documentation, the workflow, or has the re-exec + stanza in its recognised form, so a suite isolated another way is + noticed instead of being classified as not isolated. +Exit 0 if all hold; prints one line per check. +""" + +import os +import re +import subprocess +import sys + +STANZA = re.compile(r'no-docker\.sh" --check >/dev/null 2>&1 \|\|\n\s+exec "[^"\n]*/no-docker\.sh" ') +OWN = {"scripts/ci/no-docker.sh", "scripts/ci/no-docker-probe.py", "scripts/ci/tests/no-docker.test.sh", + "scripts/ci/tests/ci-isolation-order.py", ".github/workflows/ci.yml"} +SCRIPT = re.compile(r"(?= 6 and job is not None: + stripped = line.strip() + if n == 6 and stripped.startswith("- "): + step = {"name": "", "text": "", "if": False, "run": [], "in_run": False} + job["steps"].append(step) + stripped = stripped[2:] + n = 8 + if step is None: + continue + if n == 8: + step["in_run"] = False + if stripped.startswith("name:"): + step["name"] = stripped[len("name:"):].strip().strip("'\"") + elif stripped.startswith("if:"): + step["if"] = True + elif stripped.startswith("run:"): + value = stripped[len("run:"):].strip() + step["in_run"] = True + if value in ("|", "|-", ">", ">-"): + step["style"] = value[0] + else: + step["style"] = "|" + step["run"].append(value) + elif step.get("in_run"): + step["run"].append(stripped) + for job in jobs.values(): + for step in job["steps"]: + sep = " " if step.get("style") == ">" else "\n" + step["text"] = sep.join(step["run"]) + return jobs + + +def make_recipe(root, target): + try: + lines = open(os.path.join(root, "Makefile")).read().splitlines() + except OSError: + return "" + out, inside = [], False + for line in lines: + if re.match(rf"^{re.escape(target)}\s*:", line): + inside = True + continue + if inside: + if line.startswith("\t"): + out.append(line) + else: + break + return "\n".join(out) + + +def commands(text): + """The program each simple command of a shell text runs: the first word + after variable assignments and an interpreter (bash, sh, python3, exec, + env) of every command separated by a newline, ;, &&, || or |.""" + words = [] + for part in re.split(r"\n|;|&&|\|\||\|", text): + toks = part.split() + while toks and (re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", toks[0]) or toks[0] in ("exec", "env", "then", "do", "!") + or toks[0].lstrip("@") in ("bash", "sh", "python3", "/bin/bash", "/bin/sh")): + toks = toks[1:] + while toks and toks[0].startswith("-") and toks[0] != "-": + toks = toks[1:] # interpreter options such as -I or -e + if toks: + words.append(toks[0]) + return words + + +def scripts_of(root, text): + names = set() + words = commands(text) + for w in words: + if SCRIPT.fullmatch(w): + names.add(w) + for targets in re.findall(r"(?:^|[\n;&|]\s*)make((?:\s+[A-Za-z][\w-]*)+)", text): + for t in targets.split(): + names.update(w for w in commands(make_recipe(root, t).replace("\t", " ")) if SCRIPT.fullmatch(w)) + return sorted(n[2:] if n.startswith("./") else n for n in names) + + +def isolated(root, step): + """(isolated, scripts, missing scripts) of a step.""" + names = scripts_of(root, step["text"]) + missing, iso = [], False + for n in names: + try: + content = open(os.path.join(root, n), encoding="utf-8", errors="replace").read() + except OSError: + missing.append(n) + continue + if "no-docker.sh" in content and n not in ("scripts/ci/no-docker.sh", "scripts/ci/no-docker-probe.py"): + iso = True + if n == "scripts/ci/no-docker.sh": + iso = True + return iso, names, missing + + +def tracked(root): + try: + out = subprocess.run(["git", "-C", root, "ls-files", "-z"], capture_output=True, check=True).stdout + return [p for p in out.decode().split("\0") if p] + except (OSError, subprocess.CalledProcessError): + found = [] + for d, dirs, files in os.walk(root): + dirs[:] = [x for x in dirs if x != ".git"] + found += [os.path.relpath(os.path.join(d, f), root) for f in files] + return found + + +def main(): + root = os.path.abspath(sys.argv[1]) + workflow = sys.argv[2] if len(sys.argv) > 2 else os.path.join(root, ".github/workflows/ci.yml") + jobs = parse(workflow) + names = {j["name"]: jid for jid, j in jobs.items()} + for required in REQUIRED: + report(required in names, f"required job {required!r} exists") + report(not any(j["if"] or any(s["if"] for s in j["steps"]) for j in jobs.values()), + "no job or step has an if: condition") + for jid, job in jobs.items(): + first = None + late = [] + for i, step in enumerate(job["steps"]): + iso, scripts, missing = isolated(root, step) + report(not missing, f"{job['name'] or jid} / {step['name']}: the scripts it runs exist" + + (f" (missing {' '.join(missing)})" if missing else "")) + if iso and first is None: + first = step["name"] + elif not iso and first is not None: + late.append(step["name"]) + report(not late, f"{job['name'] or jid}: no step that is not isolated after the first isolated one" + + (f" ({first!r}, then {', '.join(repr(s) for s in late)})" if late else "")) + if job["name"] in REQUIRED: + report(first is not None, f"{job['name']}: has isolated steps (first: {first})") + unknown = [] + for p in tracked(root): + if p in OWN or p.endswith(".md"): + continue + try: + content = open(os.path.join(root, p), encoding="utf-8", errors="replace").read() + except OSError: + continue + if "no-docker.sh" in content and not STANZA.search(content): + unknown.append(p) + report(not unknown, "every other file that mentions no-docker.sh has the recognised re-exec stanza" + + (f" (not recognised: {' '.join(unknown)})" if unknown else "")) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/tests/no-docker.test.sh b/scripts/ci/tests/no-docker.test.sh index 308657b..e1697d2 100755 --- a/scripts/ci/tests/no-docker.test.sh +++ b/scripts/ci/tests/no-docker.test.sh @@ -3,35 +3,49 @@ # Regression tests for scripts/ci/no-docker.sh and its use by # tests/devenv/unit.sh: a mutated bin/devenv that runs Docker commands must # be refused or neutralized, whether it is passed as DEVENV or edited in -# place, and whichever way it reaches the Docker CLI or the API. +# place, and whichever way it reaches the Docker CLI or the API; and the +# isolation's limits on writes, signals, processes, namespaces and supported +# platforms must hold. # # Nothing here can change anything in the Docker daemon of this machine: # - the mutant's payload only reads (`docker version`, GET /version and # /_ping); it never removes, prunes or creates anything. A pre-flight # check refuses to run any case unless every Docker command of the -# payload is `version` and every API request a GET of /version or /_ping; +# payload is `version` and every API request a GET of /version or /_ping, +# and its sha256 is checked again before every mutant is made from it; # - its target is a canary engine of this suite: a Unix socket that # answers like Docker and records every request; two cases name the real # socket paths and must be refused at the socket; # - the harness without the containment (T-05) runs, with its canary, # behind an outer guard that leaves only the canary reachable: on macOS # a sandbox-exec profile denying every other Unix socket, TCP to -# 22/2375/2376 and beyond loopback, writes under $HOME and starting -# programs outside the sandbox; on Linux no-docker.sh itself, whose -# probe runs before the canary is started inside it; +# 22/2375/2376 and beyond loopback, writes under $HOME and to this +# suite's helpers and starting programs outside the sandbox; on Linux +# no-docker.sh itself, whose probe runs before the canary is started +# inside it; # - the harness under test (T-03, T-04) runs on Linux behind the same # guard, where it must refuse because the canary is a reachable engine. # On macOS it runs on the host itself, as a developer would run it: # macOS applies a second, different sandbox profile inside a sandbox # only if it is identical, so the harness's own isolation cannot be # tested behind the guard. Should that isolation fail, the read-only -# payload would reach the real daemon, which the cases detect. -# T-06 to T-10 run plain commands, no mutant, inside no-docker.sh: they -# check the write limits, the complete held-port list, the refusal of a -# shared $TMPDIR, preferences and TIOCSTI (macOS), and that loopback ports -# listening before the isolation started cannot be reached from inside it. +# payload would reach the real daemon, which the cases detect; +# - the helpers this suite runs outside any isolation (drive.sh, +# canary.py) and the payload are kept outside every directory the code +# under test can write, and their sha256 is checked immediately before +# each such run (T-14); +# - the other cases run plain commands that only print, connect to this +# suite's own listeners, write marker files in directories of this +# suite (and in protected places, where the write must fail; a marker +# that was written anyway is removed), send signal 0 or TERM to this +# suite's own `sleep` canaries, or sleep. # This suite itself runs no docker command. # +# T-11 to T-15 are macOS only, T-16 to T-18 Linux only (each prints INFO on +# the other platform); T-19 is static and runs on both. Each case says how +# it fails against the no-docker.sh of 38b5ebd, which these cases were +# written to catch. +# # Usage: no-docker.test.sh # HARNESS= checks another copy of tests/devenv/unit.sh instead of the # current one, with every case behind the outer guard (it is unreviewed @@ -39,7 +53,7 @@ # macOS a harness with it fails them too, as it cannot nest its sandbox. # shellcheck disable=SC2016 -# Single-quoted $... in sed and grep patterns and in the mutant stay literal. +# Single-quoted $... in sed and grep patterns and in the payloads stay literal. # shellcheck disable=SC2319 # "; result NAME $?" records the status of that condition on purpose. @@ -47,20 +61,48 @@ set -u here=$(cd -P "$(dirname "$0")" && pwd -P) root=$(cd -P "$here/../../.." && pwd -P) +nd=$root/scripts/ci/no-docker.sh +os=$(uname -s) work=$(mktemp -d "${TMPDIR:-/tmp}/no-docker-tests.XXXXXX") || exit 2 work=$(cd -P "$work" && pwd -P) [ -n "$work" ] && [ -d "$work" ] || exit 2 +# Read-only material (helpers, payload, harness trees): outside every +# directory code under test may write. On Linux it must also be visible +# inside the isolation, which hides /tmp and /var/tmp: $RUNNER_TEMP is +# mounted read-only there. +if [ "$os" = Linux ] && [ -n "${RUNNER_TEMP-}" ]; then + ro=$(mktemp -d "$RUNNER_TEMP/no-docker-tests-ro.XXXXXX") || exit 2 +else + ro=$work/ro + mkdir "$ro" || exit 2 +fi +ro=$(cd -P "$ro" && pwd -P) +helpers=$ro/helpers +mkdir "$helpers" || exit 2 +# Directories for --work: fresh and empty; on Linux under /var/tmp. +if [ "$os" = Linux ]; then + vt=$(mktemp -d /var/tmp/no-docker-tests.XXXXXX) || exit 2 +else + vt=$work/w + mkdir -m 700 "$vt" || exit 2 +fi +vt=$(cd -P "$vt" && pwd -P) home=$(cd -P "$HOME" && pwd -P) [ -n "$home" ] && [ -d "$home" ] || exit 2 nonce=$$-$(date +%s) -canary=$work/canary.sock home_marker=$home/.flowcrm-no-docker-test-$nonce -# Next to $TMPDIR: writable by the user, outside what the sandbox allows. tmp=$(cd -P "${TMPDIR:-/tmp}" && pwd -P) [ -n "$tmp" ] && [ "$tmp" != / ] || exit 2 +# Next to $TMPDIR: writable by the user, outside what the sandbox allows. outside_marker=$(dirname "$tmp")/.flowcrm-no-docker-test-$nonce tmp_marker=/private/tmp/.flowcrm-no-docker-test-$nonce -trap 'rm -f "$home_marker" "$outside_marker" "$tmp_marker"; rm -rf "$work"' EXIT +# In $TMPDIR itself, outside the run's directory. +t11_marker=$tmp/.flowcrm-no-docker-test-t11-$nonce +cleanup() { + rm -f "$home_marker" "$outside_marker" "$tmp_marker" "$t11_marker" + rm -rf "$work" "$ro" "$vt" +} +trap cleanup EXIT pass=0 fail=0 @@ -74,8 +116,16 @@ result() { # name ok(0|1) [detail] fi } +sha() { # file: its sha256 + { if command -v sha256sum >/dev/null 2>&1; then sha256sum; else shasum -a 256; fi; } <"$1" | cut -c1-64 +} + +new_work() { # a fresh, empty directory for --work + mktemp -d "$vt/w.XXXXXX" +} + # --- the canary engine -------------------------------------------------------- -cat >"$work/canary.py" <<'PY' +cat >"$helpers/canary.py" <<'PY' import json, os, socketserver, sys path, log = sys.argv[1], sys.argv[2] class H(socketserver.StreamRequestHandler): @@ -112,18 +162,21 @@ class S(socketserver.ThreadingMixIn, socketserver.UnixStreamServer): S(path, H).serve_forever() PY -# drive.sh : runs inside the outer guard. -# Starts the canary, records that the harness was started, runs it with -# DOCKER_HOST pointing at the canary and stops the canary again. -cat >"$work/drive.sh" <<'DRIVE' -dir=$1 canary=$2 log=$3 +# drive.sh : +# starts the canary (logging to /canary.log), records that the +# harness was started (/drive-started), runs it with DOCKER_HOST +# pointing at the canary and stops the canary again. +cat >"$helpers/drive.sh" <<'DRIVE' +canary_py=$1 data=$2 canary=$3 shift 3 +mkdir -p "$data" || exit 2 +: >"$data/canary.log" || exit 2 rm -f "$canary" -python3 -I "$CANARY_PY" "$canary" "$log" & +python3 -I "$canary_py" "$canary" "$data/canary.log" & cpid=$! for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do [ -S "$canary" ] && break; sleep 0.2; done [ -S "$canary" ] || { echo "HARNESS ERROR: the canary engine did not start"; kill "$cpid"; exit 2; } -: >"$dir/started" +: >"$data/drive-started" DOCKER_HOST=unix://$canary "$@" rc=$? sleep 0.5 # let the canary finish writing @@ -132,19 +185,19 @@ wait "$cpid" 2>/dev/null exit "$rc" DRIVE -export CANARY_PY=$work/canary.py - -# A Docker configuration whose context "canary" points at the canary. +# A Docker configuration whose context "canary" points at a canary socket. ctx=$(printf 'canary' | { if command -v sha256sum >/dev/null 2>&1; then sha256sum; else shasum -a 256; fi; } | cut -c1-64) -mkdir -p "$work/ctxcfg/contexts/meta/$ctx" -printf '{"Name":"canary","Metadata":{},"Endpoints":{"docker":{"Host":"unix://%s","SkipTLSVerify":false}}}\n' \ - "$canary" >"$work/ctxcfg/contexts/meta/$ctx/meta.json" +ctxcfg() { # dir canary + mkdir -p "$1/contexts/meta/$ctx" || return 1 + printf '{"Name":"canary","Metadata":{},"Endpoints":{"docker":{"Host":"unix://%s","SkipTLSVerify":false}}}\n' \ + "$2" >"$1/contexts/meta/$ctx/meta.json" +} # --- the mutant --------------------------------------------------------------- # Inserted after the first line of bin/devenv, so it runs as soon as the file # is sourced or executed. Each case appends " rc= out=" # to the log; the mutant then exits 97. -cat >"$work/payload" <<'MUT' +cat >"$helpers/payload" <<'MUT' qa_log=@LOG@ qa_canary=@CANARY@ qa_cfg=@CFG@ qa_marker=@MARKER@ qa() { # case command... qa_n=$1 @@ -184,10 +237,56 @@ qa home-write touch "$qa_marker" exit 97 MUT -# mutant : print bin/devenv with the payload after line 1. +# The payload is exactly the reviewed text: on macOS the cases T-03 and T-04 +# run without the outer guard, so their safety rests on it only reading. +# Changing it needs a new safety review and a new hash here. +payload_sha256=7fd727d79f1eb2e8e3a6f52e25da8f0fe35851b79d262384f6bbc5f5c946ed25 +helper_sums="canary.py=$(sha "$helpers/canary.py") drive.sh=$(sha "$helpers/drive.sh") payload=$(sha "$helpers/payload")" +case " $helper_sums " in + *" payload=$payload_sha256 "*) ;; + *) echo "HARNESS ERROR: the mutant payload differs from the reviewed one; refusing to run" >&2; exit 2 ;; +esac + +# verify_helpers : the helpers and the payload in are regular +# files with the sha256 recorded when this suite wrote them. Called +# immediately before every run of drive.sh and before every mutant is made. +verify_helpers() { + local f want got + for f in canary.py drive.sh payload; do + want=${helper_sums#*"$f="} + want=${want%% *} + if [ ! -f "$1/$f" ] || [ -L "$1/$f" ]; then + echo "HARNESS ERROR: $1/$f is not a regular file; refusing to run" >&2 + return 1 + fi + got=$(sha "$1/$f") + if [ "$got" != "$want" ]; then + echo "HARNESS ERROR: $1/$f changed since it was written ($got); refusing to run" >&2 + return 1 + fi + done +} + +# bare : drive.sh outside any +# isolation, only after its helpers verified. +bare() { + local dir=$1 + shift + verify_helpers "$dir" || return 2 + "$BASH" "$dir/drive.sh" "$dir/canary.py" "$@" +} + +# The paths substituted into the mutant and the profiles are plain. +for p in "$work" "$ro" "$vt" "$home_marker" "$tmp"; do + case $p in *[!A-Za-z0-9/._-]*) echo "HARNESS ERROR: unexpected character in $p; refusing to run" >&2; exit 2 ;; esac +done + +# mutant : print bin/devenv with the payload +# after line 1. mutant() { + verify_helpers "$helpers" || return 1 sed -n '1p' "$1" - sed -e "s|@LOG@|'$2'|" -e "s|@CANARY@|'$canary'|" -e "s|@CFG@|'$work/ctxcfg'|" -e "s|@MARKER@|'$home_marker'|" "$work/payload" + sed -e "s|@LOG@|'$2'|" -e "s|@CANARY@|'$3'|" -e "s|@CFG@|'$4'|" -e "s|@MARKER@|'$home_marker'|" "$helpers/payload" sed '1d' "$1" } @@ -200,17 +299,43 @@ tree() { cp "$root/tests/devenv/tally.sh" "$root/tests/devenv/port-holder.py" "$1/tests/devenv/" cp "$root/scripts/ci/no-docker.sh" "$root/scripts/ci/no-docker-probe.py" "$1/scripts/ci/" chmod +x "$1/bin/devenv" "$1/tests/devenv/unit.sh" "$1/scripts/ci/no-docker.sh" - mkdir "$1/sub" } -# guarded : run the command behind the outer guard -# (bare: on macOS without it, see the header). -guarded() { - local mode=$1 - shift - case $(uname -s)-$mode in - Darwin-bare) "$BASH" "$work/drive.sh" "$@" ;; +# run_case : run the harness +# against the mutant; sets RC, LOG, OUT, STARTED (the harness was started) +# and REQ (requests the canary engine received). +# macOS: the canary and drive.sh's records are in , outside the +# isolation; the mutant's log is in the run directory W the harness's +# no-docker.sh gets (NO_DOCKER_WORK), the only place it may write. +# Linux: the guard is `no-docker.sh --work G`; the canary, the records +# and the log are in G, the tree and helpers are visible read-only. +run_case() { + local name=$1 harness=$2 how=$3 mode=$4 t=$ro/$1 g="" w="" data canary cmd + tree "$t" "$harness" + if [ "$os" = Linux ]; then + g=$(new_work) || exit 2 + data=$g/d + LOG=$data/mutant.log + else + data=$work/$name + mkdir "$data" || exit 2 + w=$(new_work) || exit 2 + LOG=$w/mutant.log + fi + canary=$data/c + OUT=$work/$name.out + ctxcfg "$t/ctxcfg" "$canary" || exit 2 + mutant "$root/bin/devenv" "$LOG" "$canary" "$t/ctxcfg" >"$t/mutant" || exit 2 + [ "$how" = inplace ] && cp "$t/mutant" "$t/bin/devenv" + cmd=(/usr/bin/env) + [ -z "$w" ] || cmd+=("NO_DOCKER_WORK=$w") + [ "$how" = inplace ] || cmd+=("DEVENV=$t/mutant") + cmd+=("$BASH" "$t/tests/devenv/unit.sh") + rm -f "$home_marker" + case $os-$mode in + Darwin-bare) bare "$helpers" "$data" "$canary" "${cmd[@]}" ;; Darwin-guard) + verify_helpers "$helpers" || exit 2 sandbox-exec -p "(version 1) (allow default) (deny network-outbound (remote unix-socket)) @@ -218,45 +343,31 @@ guarded() { (deny network-outbound (remote ip)) (allow network-outbound (remote ip \"localhost:*\")) (deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")) -(deny file-write* (subpath \"$home\")) +(deny file-write* (subpath \"$home\") (subpath \"$ro\")) +(deny signal) +(allow signal (target same-sandbox)) (deny process-exec (literal \"/usr/bin/open\") (literal \"/usr/bin/osascript\") (literal \"/bin/launchctl\") (literal \"/usr/bin/ssh\")) (deny mach-lookup (global-name \"com.apple.coreservices.launchservicesd\") (global-name \"com.apple.coreservices.appleevents\") (global-name-regex #\"^com\\.apple\\.lsd\\.\"))" \ - "$BASH" "$work/drive.sh" "$@" + "$BASH" "$helpers/drive.sh" "$helpers/canary.py" "$data" "$canary" "${cmd[@]}" ;; - *) "$root/scripts/ci/no-docker.sh" "$BASH" "$work/drive.sh" "$@" ;; - esac -} - -# run_case : run the harness against the -# mutant; sets RC, LOG, OUT, STARTED (the harness was started inside the -# guard) and REQ (requests the canary engine received). -run_case() { - local dir=$work/$1 - tree "$dir" "$2" - LOG=$dir/mutant.log - OUT=$dir/out - : >"$LOG" - : >"$dir/canary.log" - mutant "$root/bin/devenv" "$LOG" >"$dir/mutant" - rm -f "$home_marker" - if [ "$3" = inplace ]; then - cp "$dir/mutant" "$dir/bin/devenv" - guarded "$4" "$dir/sub" "$canary" "$dir/canary.log" "$BASH" "$dir/tests/devenv/unit.sh" >"$OUT" 2>&1 - else - guarded "$4" "$dir/sub" "$canary" "$dir/canary.log" /usr/bin/env DEVENV="$dir/mutant" "$BASH" "$dir/tests/devenv/unit.sh" >"$OUT" 2>&1 - fi + *) + verify_helpers "$helpers" || exit 2 + "$nd" --work "$g" "$BASH" "$helpers/drive.sh" "$helpers/canary.py" "$data" "$canary" "${cmd[@]}" + ;; + esac >"$OUT" 2>&1 RC=$? STARTED=1 - [ -e "$dir/sub/started" ] || STARTED=0 - REQ=$(grep -c . "$dir/canary.log") + [ -e "$data/drive-started" ] || STARTED=0 + REQ=$(grep -c . "$data/canary.log" 2>/dev/null) + REQ=${REQ:-0} } # The harness without the containment: the lines that re-execute it under # no-docker.sh removed. -sed '/^# The suite sources and runs bin\/devenv/,/scripts\/ci\/no-docker.sh" "\$BASH"/d' \ - "$root/tests/devenv/unit.sh" >"$work/unit-uncontained.sh" -if cmp -s "$work/unit-uncontained.sh" "$root/tests/devenv/unit.sh" || grep -q no-docker "$work/unit-uncontained.sh" || - ! grep -q '^\. "\$devenv_script"$' "$work/unit-uncontained.sh"; then +sed '/^# The suite sources and runs bin\/devenv/,/^ exec .*scripts\/ci\/no-docker.sh" /d' \ + "$root/tests/devenv/unit.sh" >"$ro/unit-uncontained.sh" +if cmp -s "$ro/unit-uncontained.sh" "$root/tests/devenv/unit.sh" || grep -q no-docker "$ro/unit-uncontained.sh" || + ! grep -q '^\. "\$devenv_script"$' "$ro/unit-uncontained.sh"; then echo "HARNESS ERROR: could not remove the containment from tests/devenv/unit.sh" >&2 exit 2 fi @@ -265,20 +376,6 @@ harness=${HARNESS:-"$root/tests/devenv/unit.sh"} t0304=bare [ -z "${HARNESS-}" ] || t0304=guard -# The payload is exactly the reviewed text: on macOS the cases T-03 and T-04 -# run without the outer guard, so their safety rests on it only reading. -# Changing it needs a new safety review and a new hash here. -payload_sha256=7fd727d79f1eb2e8e3a6f52e25da8f0fe35851b79d262384f6bbc5f5c946ed25 -sum=$( { if command -v sha256sum >/dev/null 2>&1; then sha256sum; else shasum -a 256; fi; } <"$work/payload" | cut -c1-64) -if [ "$sum" != "$payload_sha256" ]; then - echo "HARNESS ERROR: the mutant payload differs from the reviewed one ($sum); refusing to run" >&2 - exit 2 -fi -# The paths substituted into it are plain. -for p in "$work" "$home_marker"; do - case $p in *[!A-Za-z0-9/._-]*) echo "HARNESS ERROR: unexpected character in $p; refusing to run" >&2; exit 2 ;; esac -done - # read_only : lint (in addition to the hash): every docker command is # `version` (the xargs case gets it on stdin), and no word of a command that # changes anything appears (Docker subcommands, HTTP methods other than GET). @@ -290,14 +387,14 @@ read_only() { awk '{print $NF}' | sort -u | tr '\n' ' ') [ "$words" = "version " ] } -read_only "$work/payload" || { echo "HARNESS ERROR: the mutant payload fails the read-only lint; refusing to run" >&2; exit 2; } +read_only "$helpers/payload" || { echo "HARNESS ERROR: the mutant payload fails the read-only lint; refusing to run" >&2; exit 2; } # --- T-00 the read-only check itself (payload copies are only read here) t00_bad=0 for extra in 'docker image ls' '"docker" rmi x' 'docker system df' 'qa_x=$(docker volume ls)' \ 'curl -X DELETE --unix-socket "$qa_canary" http://d/images/x' "python3 -c 's.sendall(b\"POST /x\")'" \ 'docker exec c true' 'docker --context canary info' 'curl -d x --unix-socket "$qa_canary" http://d/x'; do - { cat "$work/payload"; printf '%s\n' "$extra"; } >"$work/payload-t00" + { cat "$helpers/payload"; printf '%s\n' "$extra"; } >"$work/payload-t00" if read_only "$work/payload-t00"; then t00_bad=$((t00_bad + 1)) printf 'INFO T-00 accepted: %s\n' "$extra" @@ -307,24 +404,25 @@ done result "T-00 the read-only lint rejects a payload with a non-read-only command (9 variants)" $? # --- T-01/T-02 no-docker.sh itself -"$root/scripts/ci/no-docker.sh" --check >/dev/null 2>&1 +"$nd" --check >/dev/null 2>&1 r=$? [ "$r" -ne 0 ] result "T-01 --check fails outside the isolation" $? "(rc $r)" -NO_DOCKER_ACTIVE=1 "$root/scripts/ci/no-docker.sh" touch "$work/t02" >/dev/null 2>&1 +NO_DOCKER_ACTIVE=1 "$nd" touch "$work/t02" >/dev/null 2>&1 r=$? [ "$r" -eq 125 ] && [ ! -e "$work/t02" ] result "T-02 claimed but unproven isolation (NO_DOCKER_ACTIVE outside it): refused, command not run" $? "(rc $r)" -# --- T-06/T-07 plain commands (no mutant) inside no-docker.sh -if [ "$(uname -s)" = Darwin ]; then - "$root/scripts/ci/no-docker.sh" /bin/bash -c 'touch "$1" "$2" "$3" 2>/dev/null; cat >"$4" </dev/null; cat >"$4" </dev/null 2>&1 +EOF' _ "$home_marker" "$outside_marker" "$tmp_marker" "$w6/t06" >/dev/null 2>&1 r=$? - [ "$r" -eq 0 ] && [ "$(cat "$work/t06" 2>/dev/null)" = here-document ] && + [ "$r" -eq 0 ] && [ "$(cat "$w6/t06" 2>/dev/null)" = here-document ] && [ ! -e "$home_marker" ] && [ ! -e "$outside_marker" ] && [ ! -e "$tmp_marker" ] - result "T-06 writes only under \$TMPDIR (bash 3.2 here-documents work): not under \$HOME, next to \$TMPDIR or in /tmp" $? "(rc $r)" + result "T-06 writes only in the run's directory (bash 3.2 here-documents work): not under \$HOME, next to \$TMPDIR or in /tmp" $? "(rc $r)" rm -f "$home_marker" "$outside_marker" "$tmp_marker" # T-08: the held-port list must be complete. A netstat without its column @@ -332,15 +430,16 @@ EOF' _ "$home_marker" "$outside_marker" "$tmp_marker" "$work/t06" >/dev/null 2>& # the header and no LISTEN row is a provably empty list. mkdir -p "$work/t08" t08() { # name netstat-output expected-rc + local w printf '#!/bin/sh\nprintf "%%b" %s\n' "'$2'" >"$work/t08/netstat" chmod +x "$work/t08/netstat" - rm -f "$work/t08/ran" - PATH=$work/t08:$PATH "$root/scripts/ci/no-docker.sh" /usr/bin/touch "$work/t08/ran" >/dev/null 2>&1 + w=$(new_work) || exit 2 + PATH=$work/t08:$PATH "$nd" --work "$w" /usr/bin/touch "$w/ran" >/dev/null 2>&1 r=$? if [ "$3" -eq 0 ]; then - [ "$r" -eq 0 ] && [ -e "$work/t08/ran" ] + [ "$r" -eq 0 ] && [ -e "$w/ran" ] else - [ "$r" -eq 125 ] && [ ! -e "$work/t08/ran" ] + [ "$r" -eq 125 ] && [ ! -e "$w/ran" ] fi result "T-08 $1" $? "(rc $r)" } @@ -349,8 +448,8 @@ EOF' _ "$home_marker" "$outside_marker" "$tmp_marker" "$work/t06" >/dev/null 2>& t08 "an unparsable LISTEN row: refused, command not run" "${hdr}tcp4 0 0 garbage *.* LISTEN\n" 125 t08 "header and no LISTEN row: an empty list is accepted" "${hdr}tcp4 0 0 127.0.0.1.1 127.0.0.1.2 ESTABLISHED\n" 0 - # T-09: $TMPDIR must be a directory of its own, not /tmp. - TMPDIR=/tmp "$root/scripts/ci/no-docker.sh" /usr/bin/touch "$work/t09" >/dev/null 2>&1 + # T-09: $TMPDIR must not be /tmp. + TMPDIR=/tmp "$nd" /usr/bin/touch "$work/t09" >/dev/null 2>&1 r=$? [ "$r" -eq 125 ] && [ ! -e "$work/t09" ] result "T-09 TMPDIR=/tmp: refused, command not run" $? "(rc $r)" @@ -358,13 +457,13 @@ EOF' _ "$home_marker" "$outside_marker" "$tmp_marker" "$work/t06" >/dev/null 2>& # T-10: no preference written through cfprefsd, no keystroke injected # into the terminal (TIOCSTI), run in a pseudo-terminal of its own. t10_domain=com.flowcrm.no-docker-test-$nonce - "$root/scripts/ci/no-docker.sh" /usr/bin/defaults write "$t10_domain" k v >/dev/null 2>&1 + "$nd" /usr/bin/defaults write "$t10_domain" k v >/dev/null 2>&1 t10_pref=$(defaults read "$t10_domain" k 2>/dev/null) defaults delete "$t10_domain" >/dev/null 2>&1 rm -f "$home/Library/Preferences/$t10_domain.plist" [ -z "$t10_pref" ] result "T-10 defaults write inside the isolation changes no preference" $? - t10=$(script -q /dev/null "$root/scripts/ci/no-docker.sh" python3 -I -c ' + t10=$(script -q /dev/null "$nd" python3 -I -c ' import fcntl, os fd = os.open("/dev/tty", os.O_RDWR) try: @@ -375,9 +474,140 @@ except OSError as e: ' /dev/null | tr -d '\r') case $t10 in *"t10-denied 1"*) r=0 ;; *) r=1 ;; esac result "T-10 TIOCSTI on the controlling terminal is denied" $r "($(printf '%s' "$t10" | grep t10-))" + + # --- T-11 writes in $TMPDIR outside the run's directory are denied; in it + # they work, and the command's $TMPDIR is inside it. + # On 38b5ebd: the profile allowed the whole $TMPDIR, so the marker in + # $TMPDIR is written (and --work is not an option there: rc 127). + w11=$(new_work) || exit 2 + "$nd" --work "$w11" /bin/sh -c 'touch "$1" 2>/dev/null; touch "$2" && touch "$TMPDIR/t11"' _ "$t11_marker" "$w11/inside" >/dev/null 2>&1 + r=$? + [ "$r" -eq 0 ] && [ ! -e "$t11_marker" ] && [ -e "$w11/inside" ] && [ -e "$w11/tmp/t11" ] + result "T-11 --work: writes in the run's directory and its \$TMPDIR work, in \$TMPDIR outside it are denied" $? "(rc $r)" + rm -f "$t11_marker" + t11=$("$nd" /bin/sh -c 'touch "$1" 2>/dev/null; touch "$TMPDIR/t11" && printf "%s" "$TMPDIR"' _ "$t11_marker" 2>/dev/null) + r=$? + case $t11 in "$tmp"/no-docker.*/tmp) t11ok=0 ;; *) t11ok=1 ;; esac + [ "$r" -eq 0 ] && [ "$t11ok" -eq 0 ] && [ ! -e "$t11_marker" ] + result "T-11 without --work: \$TMPDIR is in a fresh directory of the run; \$TMPDIR itself is not writable" $? "(rc $r, TMPDIR $t11)" + rm -f "$t11_marker" + + # --- T-12 $TMPDIR allowlist: only the per-user /private/var/folders/../T. + # On 38b5ebd: a denylist accepted a user-owned directory that is not that + # one, and /private/var/tmp; the command ran there (rc 0, marker written). + mkdir -m 700 "$work/t12" "$work/t12/T" + t12v=$(mktemp -d /private/var/tmp/no-docker-tests-t12.XXXXXX) || exit 2 + t12() { # name tmpdir expected-rc + local m=$2/.t12-ran-$nonce + if [ "$3" -eq 0 ]; then + if [ -n "$2" ]; then env TMPDIR="$2" "$nd" /usr/bin/true; else env -u TMPDIR "$nd" /usr/bin/true; fi >/dev/null 2>&1 + r=$? + [ "$r" -eq 0 ] + else + env TMPDIR="$2" "$nd" /usr/bin/touch "$m" >/dev/null 2>&1 + r=$? + [ "$r" -eq 125 ] && [ ! -e "$m" ] + fi + result "T-12 $1" $? "(rc $r)" + rm -f "$m" + } + t12 "a user-owned directory named T, not the per-user one: refused" "$work/t12/T" 125 + t12 "/private/var/tmp/: refused" "$t12v" 125 + t12 "/private/var/tmp: refused" /private/var/tmp 125 + t12 "/: refused" / 125 + t12 "the per-user directory: accepted" "$tmp" 0 + t12 "TMPDIR unset (getconf DARWIN_USER_TEMP_DIR): accepted" "" 0 + rmdir "$t12v" 2>/dev/null + + # --- T-13 signals: a canary `sleep` started here, outside, cannot be + # signalled from inside (kill -0 gives EPERM) and is alive afterwards; the + # command can signal its own child and its own process group. + # On 38b5ebd: the profile had no signal rule, so kill -0 succeeded. + sleep 120 & + t13pid=$! + t13=$("$nd" python3 -I -c ' +import os, signal, subprocess, sys +try: + os.kill(int(sys.argv[1]), 0) + print("outside-reached") +except PermissionError: + print("outside-eperm") +except OSError as e: + print("outside-error", e.errno) +c = subprocess.Popen(["/bin/sleep", "30"]) +os.kill(c.pid, signal.SIGTERM) +print("own-child", c.wait()) +g = subprocess.Popen(["/bin/sleep", "30"], preexec_fn=os.setpgrp) +os.killpg(g.pid, signal.SIGTERM) +print("own-group", g.wait()) +' "$t13pid" 2>/dev/null | tr '\n' ' ') + r=$? + kill -0 "$t13pid" 2>/dev/null + alive=$? + kill -TERM "$t13pid" 2>/dev/null + wait "$t13pid" 2>/dev/null + [ "$t13" = "outside-eperm own-child -15 own-group -15 " ] && [ "$alive" -eq 0 ] + result "T-13 no signal to a process outside (EPERM, canary alive); own child and process group can be signalled" $? "($t13, canary alive rc $alive)" + + # --- T-14 integrity of what this suite runs outside any isolation. + # On 38b5ebd: the helpers were in $TMPDIR, which the sandbox allowed to + # write, so the append below succeeded. + cp "$helpers/drive.sh" "$ro/t14-target" + t14_before=$(sha "$ro/t14-target") + t14=$("$nd" /bin/sh -c 'printf "# changed\n" >>"$1" 2>/dev/null; echo "append-rc=$?"' _ "$ro/t14-target" 2>/dev/null) + [ "$t14" = "append-rc=1" ] && [ "$(sha "$ro/t14-target")" = "$t14_before" ] + result "T-14 the suite's helpers cannot be changed from inside the isolation" $? "($t14)" + # The check before a bare run, on a tampered copy: drive.sh is not run. + mkdir "$work/t14-copy" + cp "$helpers/canary.py" "$helpers/drive.sh" "$helpers/payload" "$work/t14-copy/" + printf '# tampered\n' >>"$work/t14-copy/drive.sh" + bare "$work/t14-copy" "$work/t14-data" "$work/t14-data/c" /usr/bin/touch "$work/t14-ran" >/dev/null 2>&1 + r=$? + verify_helpers "$helpers" 2>/dev/null + intact=$? + [ "$r" -eq 2 ] && [ ! -e "$work/t14-ran" ] && [ ! -e "$work/t14-data" ] && [ "$intact" -eq 0 ] + result "T-14 a changed helper is detected before a bare run, which is then not made" $? "(rc $r, intact helpers $intact)" + + # --- T-15 --work DIR: accepted for a fresh, empty, own 0700 directory + # under $TMPDIR or /private/var/tmp (writes work there and only there); + # refused otherwise. (New option: on 38b5ebd "--work" was run as the + # command, rc 127, so every case fails there.) + w15=$(new_work) || exit 2 + "$nd" --work "$w15" /bin/sh -c 'touch "$1" && { touch "$2" 2>/dev/null; true; }' _ "$w15/inside" "$work/t15-outside" >/dev/null 2>&1 + r=$? + [ "$r" -eq 0 ] && [ -e "$w15/inside" ] && [ ! -e "$work/t15-outside" ] + result "T-15 --work under \$TMPDIR: accepted, writable, nothing outside it" $? "(rc $r)" + w15v=$(mktemp -d /private/var/tmp/no-docker-tests-t15.XXXXXX) || exit 2 + "$nd" --work "$w15v" /usr/bin/touch "$w15v/inside" >/dev/null 2>&1 + r=$? + [ "$r" -eq 0 ] && [ -e "$w15v/inside" ] + result "T-15 --work under /private/var/tmp: accepted" $? "(rc $r)" + rm -rf "$w15v" + mkdir -m 700 "$work/t15" "$work/t15/full" "$work/t15/m755" "$work/t15/real" "$work/t15/real/w" + chmod 755 "$work/t15/m755" + : >"$work/t15/full/x" + ln -s "$work/t15/real" "$work/t15/link" + ln -s "$work/t15" "$work/t15/plink" + t15() { # name dir + local out + out=$("$nd" --work "$2" /bin/echo ran 2>/dev/null) + r=$? + [ "$r" -eq 125 ] && [ -z "$out" ] + result "T-15 --work $1: refused, command not run" $? "(rc $r)" + } + t15 "a symbolic link" "$work/t15/link" + t15 "with a symbolic link in its path" "$work/t15/plink/real/w" + t15 "a non-empty directory" "$work/t15/full" + t15 "mode 0755" "$work/t15/m755" + t15 "\$HOME" "$home" + t15 "/tmp" /tmp + t15 "/private/var/tmp itself" /private/var/tmp + t15 "a relative path" t15 else - echo "INFO T-06, T-08, T-09, T-10 skipped: macOS sandbox profile only (on Linux writes are not restricted, see docs/dev-environment.md)" + echo "INFO T-06, T-08 to T-15 skipped: macOS sandbox profile only" fi + +# --- T-07 loopback ports listening before the isolation cat >"$work/t07.py" <<'PY' import socket, sys, time s = socket.socket() @@ -389,7 +619,8 @@ PY python3 -I "$work/t07.py" "$work/t07.port" & t07pid=$! for _ in 1 2 3 4 5 6 7 8 9 10; do [ -s "$work/t07.port" ] && break; sleep 0.2; done -t07=$("$root/scripts/ci/no-docker.sh" python3 -I -c ' +# On Linux, no-docker.sh frames the output with ::stop-commands:: lines. +t07=$("$nd" python3 -I -c ' import socket, sys def reach(port): s = socket.socket() @@ -405,25 +636,249 @@ own = socket.socket() own.bind(("127.0.0.1", 0)) own.listen(1) print(reach(int(sys.argv[1])), reach(own.getsockname()[1])) -' "$(cat "$work/t07.port" 2>/dev/null || echo 0)" 2>/dev/null) +' "$(cat "$work/t07.port" 2>/dev/null || echo 0)" 2>/dev/null | grep -v '^::') r=$? kill "$t07pid" 2>/dev/null wait "$t07pid" 2>/dev/null [ "$r" -eq 0 ] && [ -s "$work/t07.port" ] && [ "$t07" = "refused reached" ] result "T-07 a loopback port listening before the isolation is unreachable, one opened inside it is reachable" $? "(rc $r: $t07)" +if [ "$os" = Linux ]; then + echo "INFO mount: $(mount --version 2>&1 | head -n 1)" + echo "INFO mounts outside the isolation:" + findmnt -rn -o TARGET,FSTYPE,OPTIONS 2>&1 | sed 's/^/INFO /' + echo "INFO mounts inside the isolation (and the probe's report):" + "$nd" findmnt -rn -o TARGET,FSTYPE,OPTIONS 2>&1 | sed 's/^/INFO /' + + # --- T-16 a canary inside the isolation: private pid view, no engine or + # session socket, read-only $HOME/workspace/RUNNER_TEMP, private /tmp, + # read-only mounts, masked cgroups, private IPC, loopback only. + # On 38b5ebd: no pid namespace (pid 1 is the runner's systemd, foreign + # processes visible), and $HOME and the workspace were writable. + cat >"$ro/t16.py" <<'PY' +import errno, os, socket, sys +ipc_host, out_tmp, out_shm, in_tmp, socklist, nonce = sys.argv[1:7] +targets = sys.argv[7:] +me = str(os.getuid()) +def say(good, text): + print(("t16 ok " if good else "t16 bad ") + text) +def status(pid): + st = {} + with open(f"/proc/{pid}/status") as f: + for line in f: + k, _, v = line.partition(":") + st[k] = v.strip() + return st +st = status("self") +say(len(st.get("NSpid", "").split()) >= 2, f"NSpid {st.get('NSpid')}") +s1 = status(1) +say(s1["Uid"].split()[0] == "0" and s1["Name"] == "sh", f"pid 1 is the isolation's root shell ({s1['Name']}, uid {s1['Uid'].split()[0]})") +others = [] +for d in os.listdir("/proc"): + if d.isdigit() and d != "1": + try: + s = status(d) + except OSError: + continue + if any(u != me for u in s["Uid"].split()) or s["Name"] in ("systemd", "Runner.Worker", "Runner.Listener", "dockerd", "containerd"): + others.append(f"{d}:{s['Name']}") +say(not others, f"no foreign process visible ({' '.join(others) or 'none'})") +try: + os.listdir("/proc/1/root") + say(False, "/proc/1/root is not readable") +except OSError as e: + say(e.errno in (errno.EACCES, errno.EPERM), f"/proc/1/root is not readable ({errno.errorcode.get(e.errno)})") +paths = ["/var/run/docker.sock", "/run/docker.sock", "/run/containerd/containerd.sock", "/run/dbus/system_bus_socket", + f"/run/user/{me}/bus", f"/run/user/{me}/docker.sock", "/proc/1/root/var/run/docker.sock"] +with open(socklist, "rb") as f: + paths += [os.fsdecode(p) for p in f.read().split(b"\0") if p] +reached = [] +for p in paths: + for kind in (socket.SOCK_STREAM, socket.SOCK_DGRAM): + s = socket.socket(socket.AF_UNIX, kind) + s.settimeout(2) + try: + s.connect(p) + reached.append(p) + break + except OSError: + pass + finally: + s.close() +say(not reached, f"none of {len(paths)} engine, session and host sockets connects ({' '.join(reached) or 'none'})") +for t in targets: + p = os.path.join(t, f".flowcrm-no-docker-test-t16-{nonce}") + try: + os.close(os.open(p, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)) + os.unlink(p) + say(False, f"write in {t} fails (written, removed again)") + except OSError as e: + say(e.errno == errno.EROFS, f"write in {t} fails with EROFS ({errno.errorcode.get(e.errno)})") +say(not os.path.exists(out_tmp), "a file in the caller's /tmp is not visible") +say(not os.path.exists(out_shm), "a file in the caller's /dev/shm is not visible") +try: + os.close(os.open(in_tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)) + say(True, "the private /tmp is writable") +except OSError as e: + say(False, f"the private /tmp is writable ({e})") +allowed = {"/tmp", "/var/tmp", "/dev/shm", "/run", "/var/run", "/dev/mqueue", "/proc", "/dev", "/dev/pts", + os.environ.get("NO_DOCKER_RUN_DIR", "")} +rw = [] +cgroup_fs = "none" +with open("/proc/self/mountinfo") as f: + rows = [line.split() for line in f] +for r in rows: + point = r[4].replace("\\040", " ") + try: + fd = os.open(point, os.O_PATH | os.O_NOFOLLOW) + except OSError: + continue + try: + mnt = [l.split()[1] for l in open(f"/proc/self/fdinfo/{fd}") if l.startswith("mnt_id:")][0] + finally: + os.close(fd) + if mnt != r[0]: + continue + if point == "/sys/fs/cgroup": + cgroup_fs = r[r.index("-") + 1] + if point not in allowed and "ro" not in r[5].split(","): + rw.append(point) +say(not rw, f"every other reachable mount is read-only ({' '.join(rw) or 'none writable'})") +say(cgroup_fs != "cgroup2" and not os.access("/sys/fs/cgroup/cgroup.procs", os.W_OK) + and not os.access("/sys/fs/cgroup/cgroup.kill", os.W_OK), f"/sys/fs/cgroup is masked ({cgroup_fs})") +say(os.readlink("/proc/self/ns/ipc") != ipc_host, f"private IPC namespace ({os.readlink('/proc/self/ns/ipc')}, caller {ipc_host})") +names = sorted(n for _, n in socket.if_nameindex()) +say(names == ["lo"], f"only loopback ({' '.join(names)})") +PY + t16_tmp=/tmp/.flowcrm-no-docker-test-t16-$nonce + t16_shm=/dev/shm/.flowcrm-no-docker-test-t16-$nonce + t16_in=/tmp/.flowcrm-no-docker-test-t16-in-$nonce + : >"$t16_tmp" + : 2>/dev/null >"$t16_shm" + # Sockets of the host, found here independently of no-docker.sh. + { python3 -I -c ' +import sys +with open("/proc/net/unix", "rb") as f: + next(f) + for line in f: + p = line.split(None, 7) + if len(p) == 8 and p[7].startswith(b"/"): + sys.stdout.buffer.write(p[7].rstrip(b"\n") + b"\0") +' + find "$home" -xdev -type s -print0 2>/dev/null + } >"$ro/t16-sockets" + t16_targets=("$home") + for v in GITHUB_WORKSPACE RUNNER_TEMP; do [ -z "${!v-}" ] || t16_targets+=("${!v}"); done + [ ! -d /opt/hostedtoolcache ] || t16_targets+=(/opt/hostedtoolcache) + t16=$("$nd" python3 -I "$ro/t16.py" "$(readlink /proc/self/ns/ipc)" "$t16_tmp" "$t16_shm" "$t16_in" \ + "$ro/t16-sockets" "$nonce" "${t16_targets[@]}" 2>&1) + r=$? + printf '%s\n' "$t16" | grep '^t16 ' | sed 's/^/INFO /' + t16_left="" + for t in "${t16_targets[@]}"; do + if [ -e "$t/.flowcrm-no-docker-test-t16-$nonce" ]; then + t16_left="$t16_left $t" + rm -f "$t/.flowcrm-no-docker-test-t16-$nonce" + fi + done + [ ! -e "$t16_in" ] || t16_left="$t16_left /tmp" + t16_ok=$(printf '%s\n' "$t16" | grep -c '^t16 ok') + [ "$r" -eq 0 ] && ! printf '%s\n' "$t16" | grep -q '^t16 bad' && [ "$t16_ok" -ge 12 ] && [ -z "$t16_left" ] + result "T-16 inside the isolation: private pids, no reachable socket, read-only \$HOME/workspace/RUNNER_TEMP, private /tmp, read-only mounts, masked cgroups, private IPC" $? \ + "(rc $r, $t16_ok ok, markers left in:${t16_left:- none})" + rm -f "$t16_tmp" "$t16_shm" "$t16_in" + + # --- T-17 no process survives the isolated command: a `setsid -f` canary + # with a unique marker, after a normal end and after the isolation itself + # (unshare) is killed. Checked from outside, after no-docker.sh returned. + # On 38b5ebd: no pid namespace and no --kill-child, the canary survives; + # and there is no unshare process to kill (it exec'd), which fails too. + t17m=t17-$nonce + "$nd" /bin/sh -c 'setsid -f /bin/sh -c "sleep 60" "$1-in" /dev/null 2>&1; exit 0' _ "$t17m" >/dev/null 2>&1 + r=$? + for _ in 1 2 3 4 5 6 7 8 9 10; do pgrep -f -- "$t17m-in" >/dev/null || break; sleep 0.5; done + pgrep -f -- "$t17m-in" >/dev/null + left=$? + pkill -TERM -f -- "$t17m-in" 2>/dev/null + [ "$r" -eq 0 ] && [ "$left" -ne 0 ] + result "T-17 a daemonised process does not outlive the isolated command" $? "(rc $r, survivor $([ "$left" -eq 0 ] && echo yes || echo no))" + + t17c=t17c-$nonce + "$nd" /bin/sh -c 'setsid -f /bin/sh -c "sleep 60" "$1-in" /dev/null 2>&1; sleep 60' _ "$t17c" >/dev/null 2>&1 & + t17pid=$! + for _ in $(seq 1 240); do pgrep -f -- "$t17c-in" >/dev/null && break; sleep 0.5; done + started=no + pgrep -f -- "$t17c-in" >/dev/null && started=yes + upid="" + for p in $(pgrep -x unshare); do + q=$p + while [ -n "$q" ] && [ "$q" -gt 1 ]; do + q=$(ps -o ppid= -p "$q" 2>/dev/null | tr -d ' ') + if [ "$q" = "$t17pid" ]; then upid=$p; break 2; fi + done + done + [ -z "$upid" ] || sudo -n kill -KILL "$upid" + wait "$t17pid" 2>/dev/null + for _ in 1 2 3 4 5 6 7 8 9 10; do pgrep -f -- "$t17c-in" >/dev/null || break; sleep 0.5; done + pgrep -f -- "$t17c-in" >/dev/null + left=$? + pkill -TERM -f -- "$t17c" 2>/dev/null + [ "$started" = yes ] && [ -n "$upid" ] && [ "$left" -ne 0 ] + result "T-17 killing the isolation (unshare) kills every process inside it" $? \ + "(canary started $started, unshare ${upid:-not found}, survivor $([ "$left" -eq 0 ] && echo yes || echo no))" + + # --- T-18 unsupported use is refused: uid 0 (a real sudo run) and a Linux + # that is not a GitHub-hosted runner. The command is `true`. + # On 38b5ebd: neither was checked, so every case ran `true` (rc 0). + t18() { # name expected-message command... + local name=$1 msg=$2 err + shift 2 + err=$("$@" 2>&1 >/dev/null) + r=$? + [ "$r" -eq 125 ] && printf '%s' "$err" | grep -q -- "$msg" + result "T-18 $name: refused" $? "(rc $r)" + } + t18 "uid 0 (sudo)" "running as root" sudo -n "$nd" /usr/bin/true + t18 "RUNNER_ENVIRONMENT unset" "GitHub-hosted" env -u RUNNER_ENVIRONMENT "$nd" /usr/bin/true + t18 "RUNNER_ENVIRONMENT=self-hosted" "GitHub-hosted" env RUNNER_ENVIRONMENT=self-hosted "$nd" /usr/bin/true + t18 "GITHUB_ACTIONS unset" "GitHub-hosted" env -u GITHUB_ACTIONS "$nd" /usr/bin/true + t18 "GITHUB_WORKSPACE outside \$HOME" "GitHub-hosted" env GITHUB_WORKSPACE=/var/tmp "$nd" /usr/bin/true + # Without a Runner.Worker ancestor: started through setsid -f, so it is + # reparented away from this job's process tree; its status comes back in + # a file. + setsid -f /bin/sh -c '"$1" /usr/bin/true >/dev/null 2>"$2.err"; echo $? >"$2.tmp"; mv "$2.tmp" "$2"' _ "$nd" "$work/t18.rc" + for _ in $(seq 1 240); do [ -s "$work/t18.rc" ] && break; sleep 0.5; done + r=$(cat "$work/t18.rc" 2>/dev/null) + [ "$r" = 125 ] && grep -q "no Runner.Worker ancestor" "$work/t18.rc.err" 2>/dev/null + result "T-18 no Runner.Worker ancestor: refused" $? "(rc ${r:-none})" +else + echo "INFO T-16, T-17, T-18 skipped: Linux (GitHub-hosted runner) only" +fi + +# --- T-19 CI order (static): no step that is not isolated follows an +# isolated one in a job; the required jobs exist; no if: conditions. +# On 38b5ebd: "Allocator unit tests" came before the real-Docker steps and +# the guard suites before ShellCheck and actionlint. +t19=$(python3 -I "$here/ci-isolation-order.py" "$root" 2>&1) +r=$? +printf '%s\n' "$t19" | grep -v '^ci-isolation-order: ok' | sed 's/^/INFO /' +[ "$r" -eq 0 ] +result "T-19 isolated steps are the last steps of their CI jobs; required jobs exist; no if: conditions" $? "(rc $r)" + # --- T-03/T-04 the harness against the mutant check_safe() { # name # Refused (125, the mutant never ran) or run isolated (97, every case ran # and none reached an engine); either way the canary engine got nothing. local cases answered - cases=$(grep -c ' rc=' "$LOG") - answered=$(grep -c 'canary-engine' "$LOG") + cases=$(grep -c ' rc=' "$LOG" 2>/dev/null) + cases=${cases:-0} + answered=$(grep -c 'canary-engine' "$LOG" 2>/dev/null) + answered=${answered:-0} [ "$REQ" -eq 0 ] && [ "$answered" -eq 0 ] result "$1: no request reached an engine" $? "(canary requests $REQ, answered cases $answered, rc $RC)" [ "$STARTED" -eq 1 ] result "$1: the harness was started" $? - if [ "$(uname -s)" = Darwin ] && [ "$t0304" = bare ]; then + if [ "$os" = Darwin ] && [ "$t0304" = bare ]; then # The harness nests its own sandbox inside the guard and runs the mutant. [ "$RC" -eq 97 ] && [ "$cases" -ge 20 ] result "$1: the mutant ran to its end inside the harness's isolation" $? "(rc $RC, $cases cases)" @@ -433,29 +888,34 @@ check_safe() { # name grep -q '^no-docker: refused: inside no-docker.sh, but the isolation check fails' "$OUT" result "$1: the harness refused before the mutant ran (an engine is reachable)" $? "(rc $RC, $cases cases)" fi - ! grep -E '^(real-socket-cli|real-socket-curl) rc=0 ' "$LOG" >/dev/null + ! grep -E '^(real-socket-cli|real-socket-curl) rc=0 ' "$LOG" >/dev/null 2>&1 result "$1: the real Docker socket paths were not reached" $? [ ! -e "$home_marker" ] result "$1: nothing written under \$HOME" $? rm -f "$home_marker" } run_case t03 "$harness" devenv "$t0304" +shown=("t03 $LOG $OUT") check_safe "T-03 mutant as DEVENV" run_case t04 "$harness" inplace "$t0304" +shown+=("t04 $LOG $OUT") check_safe "T-04 mutant edited into bin/devenv" # --- T-05 control: the same check catches a harness without containment -run_case t05 "$work/unit-uncontained.sh" inplace guard -answered=$(grep -c 'canary-engine' "$LOG") +run_case t05 "$ro/unit-uncontained.sh" inplace guard +shown+=("t05 $LOG $OUT") +answered=$(grep -c 'canary-engine' "$LOG" 2>/dev/null) +answered=${answered:-0} [ "$STARTED" -eq 1 ] && [ "$REQ" -gt 0 ] && [ "$answered" -gt 0 ] && [ "$RC" -eq 97 ] result "T-05 control: without the containment the mutant's commands reach the canary engine" $? "(canary requests $REQ, answered cases $answered, rc $RC)" echo -for f in t03 t04 t05; do +for s in "${shown[@]}"; do + read -r f log out <<<"$s" printf 'INFO %s cases:\n' "$f" - sed 's/^/INFO /' "$work/$f/mutant.log" 2>/dev/null + sed 's/^/INFO /' "$log" 2>/dev/null printf 'INFO %s harness output (last lines):\n' "$f" - tail -n 8 "$work/$f/out" 2>/dev/null | sed 's/^/INFO /' + tail -n 8 "$out" 2>/dev/null | sed 's/^/INFO /' done echo echo "TOTAL $((pass + fail)) passed=$pass failed=$fail" From 45498a8a7bea5fb64e12c3c3c859283974bcdbc7 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 09:02:10 +0100 Subject: [PATCH 12/77] ci: run the isolated test steps last in their jobs Code under test runs only in the last steps of Repository guard (the guard regression suites and the isolation tests, after ShellCheck and actionlint) and of Dev environment (the allocator unit tests, after the container scan), so no step that is not isolated runs after it. Job names, the job set, triggers and permissions are unchanged. --- .github/workflows/ci.yml | 41 ++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5479da..1630f01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,30 +28,15 @@ jobs: - name: Reject tracked local-only files run: scripts/ci/guard-tracked-files.sh - - name: Run guard regression tests - run: scripts/ci/tests/guard-tracked-files.test.sh - - name: Require a digest-pinned Dockerfile frontend run: scripts/ci/guard-dockerfile-frontend.sh - - name: Run Dockerfile frontend guard regression tests - run: scripts/ci/tests/guard-dockerfile-frontend.test.sh - - name: Check the container scan exceptions against their policy run: scripts/ci/guard-trivyignore.sh - - name: Run container scan exception guard regression tests - run: scripts/ci/tests/guard-trivyignore.test.sh - - name: Keep the edge configuration free of ACME server, PKI, issuers and mutable admin run: scripts/ci/guard-edge-config.sh - - name: Run edge configuration guard regression tests - run: scripts/ci/tests/guard-edge-config.test.sh - - - name: Run test isolation regression tests - run: scripts/ci/tests/no-docker.test.sh - - name: ShellCheck run: >- docker run --rm --network none -v "$PWD:/mnt:ro" -w /mnt @@ -68,6 +53,24 @@ jobs: rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667 -color + # Code under test runs only in the steps from here to the end of the + # job, each isolated by scripts/ci/no-docker.sh; no step that is not + # isolated may follow them (checked by scripts/ci/tests/ci-isolation-order.py). + - name: Run guard regression tests + run: scripts/ci/tests/guard-tracked-files.test.sh + + - name: Run Dockerfile frontend guard regression tests + run: scripts/ci/tests/guard-dockerfile-frontend.test.sh + + - name: Run container scan exception guard regression tests + run: scripts/ci/tests/guard-trivyignore.test.sh + + - name: Run edge configuration guard regression tests + run: scripts/ci/tests/guard-edge-config.test.sh + + - name: Run test isolation regression tests + run: scripts/ci/tests/no-docker.test.sh + # The application checks run in this required job on purpose: a separate # job that is not a required check could fail (or be skipped) without # blocking a merge. @@ -109,9 +112,6 @@ jobs: - name: make help and make doctor run: make help && make doctor - - name: Allocator unit tests - run: tests/devenv/unit.sh - - name: Allocator integration tests (real Docker) run: tests/devenv/integration.sh @@ -131,3 +131,8 @@ jobs: # docs/security/container-vulnerabilities.md). - name: Container vulnerability scan (Trivy) run: tests/app/scan.sh + + # Code under test runs only from here to the end of the job, isolated by + # scripts/ci/no-docker.sh (order checked by scripts/ci/tests/ci-isolation-order.py). + - name: Allocator unit tests + run: tests/devenv/unit.sh From 6d484c9215775a7e0fc6bb9519a5ebe1a2b14f54 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 09:02:10 +0100 Subject: [PATCH 13/77] docs(dev-environment): describe the isolation's platforms, properties and residual risks Supported platforms (macOS developer machines, Linux on GitHub-hosted runners only), each enforced property and how the probe verifies it, --work, the CI order, the regression cases, the accepted residual risks and the interim mutation-testing policy. --- docs/dev-environment.md | 426 ++++++++++++++++++++++++++++++---------- 1 file changed, 318 insertions(+), 108 deletions(-) diff --git a/docs/dev-environment.md b/docs/dev-environment.md index 318233d..3e175b1 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -417,116 +417,326 @@ $ make test-app # application: the four suites `tests/devenv/unit.sh` and the guard regression suites in `scripts/ci/tests/` run code under test, including a modified or mutated -copy of it (`DEVENV=`, `GUARD=`, `ADAPTED=` or an edited file). They run it -only where no container engine can be reached: each re-executes itself under -`scripts/ci/no-docker.sh`, so a command in that code cannot touch the -images, containers, volumes, networks or build cache of the machine, however -it reaches Docker (the CLI on `PATH` or by absolute path, a nested shell or -command substitution, `-H`, `--context` or an inherited `DOCKER_HOST`, -`curl --unix-socket` or a raw socket). - -- macOS: `sandbox-exec` denies: - - every connect to a Unix socket; - - TCP beyond loopback, and to loopback ports 22, 2375 and 2376; - - TCP to every loopback port that was already listening when the suite - started, such as a local development server, an IDE, a published - container port or a socket proxy. The list comes from `netstat`. If - `netstat` fails, prints no column header or prints a LISTEN row that - cannot be parsed, the suite is refused (exit 125). An empty list is accepted only when - `netstat` printed its header and no LISTEN row; - - writes anywhere except under `$TMPDIR` and `/dev`, and in `/tmp` the - `sh-thd*` here-document files of bash 3.2, which it creates there - whatever `$TMPDIR` is. This covers `$HOME`, where Docker Desktop keeps - its disk image, and places whose files are run or read outside the - sandbox later: Homebrew's prefix, `/Applications` and other programs' - temporary files in `/tmp`. `$TMPDIR` must - therefore be a directory of its own. If it is `/`, `/tmp`, under `/tmp` - or under `$HOME`, the suite is refused (exit 125). The macOS default - under `/var/folders` is fine; - - hard links do not get around this. Measured: a link from `$TMPDIR` to - a file in a write-denied place is refused; - - preference writes through `cfprefsd` (`defaults write`), which would - otherwise reach `~/Library/Preferences` although `$HOME` is - write-denied; - - `TIOCSTI` on the terminal, which would otherwise let a command inject - keystrokes into the shell that started the suite; - - `open`, `osascript`, `launchctl` and `ssh`, job submission to launchd, - and access to Launch Services and Apple Events, so that no program is - started outside the sandbox. A re-signed copy of one of these programs - can run, but launchd, Launch Services and Apple Events still refuse it, - and the network routes of `ssh` are denied above. - - The profile filters these routes; it is not a complete jail. Known - residual routes: - - loopback ports that the suite opens itself stay reachable. So does a - service that starts listening after the suite started; the probe asks - each listening port whether it is an engine only at the start; - - a command can still create `sh-thd*` files or directories in `/tmp`. - bash creates its own here-document files exclusively, so at worst a - later here-document of another shell fails; - - signals to the user's processes outside the sandbox are allowed. A - command could stop Docker Desktop, but not change its objects; - - login items and background tasks are registered through Launch - Services, Apple Events or launchd, which are denied. Their other - interfaces were not measured; - - the write check next to `$TMPDIR` proves nothing when the directory - holding `$TMPDIR` is not writable by the user anyway. -- Linux: `sudo unshare` gives a network namespace with only loopback and an - empty tmpfs over `/run`; the suite then runs as the calling user without - supplementary groups (no `docker` group) and with `no_new_privs`. This - needs passwordless sudo. -- `DOCKER_*` and similar variables are removed and `DOCKER_CONFIG` is an - empty directory. -- Before the suite runs, `scripts/ci/no-docker-probe.py` checks from inside - that the mechanism is in effect and that no engine answers on any endpoint - it finds: the usual socket paths, the Docker contexts, `DOCKER_HOST`, the - sockets under `/run`, `/var/run`, `/tmp`, `$TMPDIR` and `~/.docker`, - ports 2375 and 2376 and every listening loopback port. If the isolation cannot be applied or proven, the - suite does not run (exit 125). -- Setuid programs cannot run inside, so `sudo` is unavailable. The U-16 - check that another local user cannot read a secret therefore prints INFO - and is not counted, in CI as well as locally; the U-16 mode checks still - run. -- On Linux the probe finds an engine socket owned by the calling user only - in the places listed above. Writes are not restricted on Linux, so a - mutant could change files that run later outside the isolation. CI - runners are discarded after each job; on a developer machine, run - mutants only with read-only payloads (see below). - -- macOS applies a sandbox inside another only if the profiles are - identical, so the suites cannot run inside a different sandbox: they are - refused (exit 125). - -`scripts/ci/tests/no-docker.test.sh` checks this with a mutated `bin/devenv` -whose commands reach Docker in more than 20 ways, both as `DEVENV` and edited -into the file. - -- The mutant only reads (`docker version`, `GET /version`); a pre-flight - check refuses to run if it contains anything else. -- Its target is a canary engine of the test, which answers like Docker and - records every request. Two cases name the real socket paths. -- On macOS, `unit.sh` runs on the host as a developer would run it. The - mutant must run to its end with no request reaching the canary and no - answer from the real socket paths. -- On Linux, `unit.sh` runs behind `no-docker.sh`, where the canary is a - reachable engine, so `unit.sh` must refuse. -- The control runs the same mutant under a copy of `unit.sh` without the - re-execution, behind an outer guard that leaves only the canary - reachable. The mutant must reach the canary. -- Without a mutant, plain commands inside `no-docker.sh` must not reach a - loopback port that was listening before it started, while a port they - open themselves stays reachable. On macOS they must also be unable to - write under `$HOME`, next to `$TMPDIR` or in `/tmp`, while bash 3.2 - here-documents still work. They must not be able to write a preference - or use `TIOCSTI` either. An incomplete port list and `TMPDIR=/tmp` must - be refused. - -Mutation testing: +copy of it (`DEVENV=`, `GUARD=`, `ADAPTED=` or an edited file). That code +runs with the caller's user ID and may be any shell, Python or program on +the machine: it may fork, daemonise, write files, open sockets and send +signals. Each suite therefore re-executes itself under +`scripts/ci/no-docker.sh`. The isolation must keep the code away from: + +- the container engines and their objects (images, containers, volumes, + networks, build cache), including those of other projects; +- the user's environment: files in `$HOME` and in the checkout, shell and + tool configuration, user-session IPC and the user's other processes; +- later steps of the same CI job. + +The person or job that starts a suite is trusted. The checks of the +environment described below prevent accidental use where the isolation is +not supported; they are not a boundary against the caller. Kernel and +sandbox vulnerabilities and CPU or memory exhaustion are out of scope. + +Supported platforms: + +| Platform | Support | Mechanism | +|---|---|---| +| macOS developer machine | supported | `sandbox-exec` profile | +| Linux on a GitHub-hosted Actions runner | supported | `sudo -n unshare` namespaces | +| Linux anywhere else (developer machines, self-hosted runners, containers) | refused (exit 125): "Linux is supported on GitHub-hosted CI runners only" | — | +| other systems | refused (exit 125) | — | +| any platform as uid 0 | refused (exit 125) | — | + +Each property below is enforced by the sandbox or the kernel and checked +from inside by `scripts/ci/no-docker-probe.py` before the command starts. If +any check fails, a tool is missing, a mount fails, the profile is rejected or +the command was not started inside the isolation, the command does not run +and the exit status is 125. + +On both platforms the variables that select an engine (`DOCKER_*`, +`CONTAINER_HOST`, `PODMAN_*`, `COMPOSE_*` and similar) are removed, +`DOCKER_CONFIG` is an empty directory and `$TMPDIR` is a directory of the +run. The probe then asks every endpoint it can find whether a container +engine answers there: the usual socket paths, the Docker contexts, +`DOCKER_HOST`, sockets under `/run`, `/var/run`, `/tmp`, `$TMPDIR` and +`~/.docker`, ports 2375 and 2376 and every listening TCP port. The walk +of those directories stops at 20,000 entries. On Linux that refuses the +run. On macOS, where every Unix socket connect is denied anyway, the probe +prints an INFO line instead. + +#### macOS + +`$TMPDIR` must be the per-user temporary directory that macOS creates, +`/private/var/folders///T`, owned by the user with mode 0700. If +`TMPDIR` is unset, `getconf DARWIN_USER_TEMP_DIR` is used. Any other +directory, such as `/tmp`, `/private/var/tmp`, `/`, Homebrew's prefix or a +directory of the user's own, is refused. Each run gets a fresh directory +there, mode 0700. The command's `$TMPDIR` is inside it. + +The `sandbox-exec` profile denies: + +- connecting to any Unix socket. The probe checks this with a socket of its + own: EPERM; +- TCP beyond loopback, and to loopback ports 22, 2375 and 2376. The probe + checks 2375 and a non-loopback address; +- TCP to every port that was listening when the run started, on any address + (a development server, an IDE, a published container port or a socket + proxy). The list comes from `netstat`; if `netstat` fails, prints no + column header or prints a LISTEN row that cannot be parsed, the run is + refused. The probe connects to each held port; +- file writes anywhere except in the run's directory, `/dev/null`, + `/dev/zero`, `/dev/tty`, `/dev/dtracehelper`, `/dev/fd/*` and the + `sh-thd*` here-document files of bash 3.2, which bash creates in `/tmp` + whatever `$TMPDIR` is. The probe checks that `$HOME`, the user's + temporary directory, `/tmp` and `/private/var/tmp` are not writable and + that `$TMPDIR` is. This `/dev` list was measured with bash 3.2, python3, + `mktemp`, `/usr/bin/git`, `tests/devenv/unit.sh` and the guard suites. + Writing to `/dev/random`, `/dev/urandom` and other terminals' `/dev/ttys*` + is denied. `/usr/bin/git` reports that it cannot write the `xcrun_db` + cache in the user's temporary directory and still works. That cache is + read by `xcrun` outside the sandbox, so it stays write-denied; +- signals to any process outside the sandbox. Processes of the run can + signal each other, including process groups. The probe sends signal 0 to + `no-docker.sh` outside: EPERM; +- preference writes through `cfprefsd` and `TIOCSTI` on the terminal; +- `open`, `osascript`, `launchctl` and `ssh`, job submission to launchd, + Launch Services and Apple Events; +- lookups of the services that register login items and background tasks + (`com.apple.xpc.smd`, `com.apple.backgroundtaskmanagement*`, + `com.apple.coreservices.sharedfilelistd*`) and of Docker Desktop's own + services (`com.docker.*`, `com.electron.dockerdesktop.*`). + +Setuid programs cannot run in the sandbox, so `ps` is an ad-hoc signed copy +and `sudo` is unavailable. The U-16 check that another local user cannot +read a secret therefore prints INFO and is not counted; the U-16 mode checks +still run. macOS applies a sandbox inside another only if the profiles are +identical, so the suites cannot run inside a different sandbox: they are +refused. + +#### Linux (GitHub-hosted runners) + +`no-docker.sh` refuses unless: + +- the caller is not uid 0 (real or effective). This is checked before + anything else; +- `GITHUB_ACTIONS=true` and `RUNNER_ENVIRONMENT=github-hosted`; +- a `Runner.Worker` process (by `comm` and executable name) is an ancestor + of the caller. An unreadable ancestor counts as a refusal; +- `$GITHUB_WORKSPACE` and `$RUNNER_TEMP` are directories under `$HOME`; +- `sudo -n` works and `unshare`, `setpriv`, `setsid`, `ip`, `mount`, `find`, + `stat` and `python3` exist. + +Before entering, the caller's side lists every Unix socket file on the +local file systems (`find -xdev -type s` per mount) and every bound +pathname socket of the host's network namespace (`/proc/net/unix`, which is +what `ss -xl` shows). The list has a 180-second budget; when it runs out, +the run is refused instead of using a partial list. The time it took is +printed with each run. Inside, the list is a read-only file. + +The command then runs under +`sudo -n unshare --net --mount --pid --fork --kill-child --mount-proc --ipc` +with private mount propagation. The namespace's pid 1 is a root shell that +does not exec: + +1. It brings up loopback in the new network namespace. +2. It mounts an empty tmpfs over `/run` (and `/var/run` when that is not a + symbolic link), `/tmp`, `/var/tmp` and `/dev/shm`, a fresh mqueue on + `/dev/mqueue` and a read-only tmpfs over `/sys/fs/cgroup`. The run's + directory, which is under `/tmp` (or under `/var/tmp` with `--work`), is + bind-mounted aside first and moved back into the new tmpfs, so the + original `/tmp` is not visible. +3. It remounts every other mount reachable by path read-only. A bind + remount changes only the namespace's copy and keeps the other per-mount + flags. This covers `$HOME` with the workspace, `$RUNNER_TEMP`, + `$GITHUB_ENV`, `$GITHUB_PATH` and `~/.docker`, as well as + `/opt/hostedtoolcache` and every other mount. +4. It runs the command as a child: `setpriv` with the caller's uid and gid, + no supplementary groups, `no_new_privs`, and empty inheritable and + bounding capability sets; then `setsid --wait`, then `env -i` with an + allowlist of variables (`PATH`, `HOME`, `USER`, `LOGNAME`, `SHELL`, + `LANG`, `LANGUAGE`, `LC_*`, `TERM`, `TZ`, `CI`, `DEVENV*`, `GUARD*`, + `ADAPTED`, `HARNESS` and `FLOWCRM_*`). No session bus, display, agent, + runner file-command or token variable reaches the command. +5. It waits for the command, reaping orphans meanwhile, and exits with the + command's status. When pid 1 exits, the kernel kills every process left + in the namespace, including daemonised ones. If `unshare` itself is + killed (a cancelled job or a timeout), `--kill-child` kills pid 1. pid 1 + makes no credential change, so that signal stays armed. + +The probe checks from inside: + +- no uid is 0; every capability set is empty; `NoNewPrivs` is 1; there are + no supplementary groups; +- the network, mount, pid and IPC namespaces differ from the caller's; +- the process is in a pid namespace below the caller's (`NSpid`), and + `/proc` belongs to it; +- pid 1 is the root shell of the isolation and every other visible process + has the caller's uid. None is a runner, init or engine process, so + `/proc//root` and `nsenter` have no target outside; +- the process is in a session of its own without a controlling terminal, + so `TIOCSTI` has no target; +- loopback is the only interface; +- `/tmp`, `/var/tmp`, `/dev/shm` and `/run` are tmpfs, and `/dev/mqueue` an + mqueue, on devices other than the caller's. `/sys/fs/cgroup` is a + read-only tmpfs, and no cgroup file is writable; +- every mount reachable by path is read-only, except the private file + systems above, the run's directory, `/proc` and `/dev`. This is read from + `/proc/self/mountinfo` and the mount ID of each path. Creating a file in + `$HOME`, `$GITHUB_WORKSPACE` and `$RUNNER_TEMP` must fail as well; +- the working directory is reachable by its path; +- there is no socket in the private directories, and none of the listed + Unix sockets accepts a stream, seqpacket or datagram connection. + Pathname sockets on a read-only mount would still accept one, which is + why every one of them is tried. + +The output between the start and the end of the isolated command is framed +with `::stop-commands::`, so the command cannot issue +workflow commands. The token is never passed inside. + +#### Run directory (`--work DIR`) + +`no-docker.sh --work DIR COMMAND` uses `DIR` as the run's directory +instead of a temporary one, and keeps it afterwards. It is the command's +only writable directory and holds its `$TMPDIR`. + +`DIR` must: + +- be given as an absolute, canonical path with no symbolic link in it; +- be a directory of the caller with mode 0700; +- be empty; +- not be or contain `$HOME`; +- be under `$TMPDIR` or `/private/var/tmp` on macOS, and under `/var/tmp` + on Linux. + +Otherwise the run is refused. `tests/devenv/unit.sh` passes +`NO_DOCKER_WORK=` on as `--work`. Whatever the command leaves in +`DIR` is untrusted data: a later reader must not follow symbolic links, +must accept regular files only, must limit their size and must not execute +them. + +#### Accepted residual risks + +- **macOS leftover processes.** Background processes of the command can + outlive it. They stay inside the sandbox, with every limit above, but + have an unbounded time for the routes below. Killing the run's process + group or session afterwards is a follow-up. +- **Late listeners.** A loopback port that starts listening after the run + started (on macOS) is reachable, as are the ports the command opens + itself. This includes a VM's port forward that starts later, or one + started while a leftover process still runs. The probe asks each + listening port whether it is an engine only at the start. +- **Login items and launchd environment through copies (macOS).** The + services that register login items and background tasks are denied. A + re-signed copy of `launchctl` loses its entitlements. Whether such a copy + can still `setenv` in the user's launchd domain was not measured. +- **Terminals and the pasteboard (macOS).** Writes to other terminals' + devices are denied. The pasteboard (`pbcopy`) is not; using it needs a + person to paste. +- **Writes through other processes (macOS).** The file-write limit covers + the command's own writes. Writes made for it by system services are + limited by the launchd, Launch Services, Apple Events, preference and + login-item denies above, not by the file rule. Keychain items are data, + not code. +- **Sockets created after the list (Linux).** A pathname socket that + appears on a read-only mount after the list was made, from a process + outside the namespace, is not tried. On a GitHub-hosted runner no such + service runs during the isolated steps. +- **Shared keyring and readable files (Linux).** The per-user kernel + keyring is shared. Files such as `~/.docker/config.json` stay readable, + but there is no network to use them on. +- **Orphan cleanup after a kill (Linux).** `env -i` drops + `RUNNER_TRACKING_ID`, so the runner's end-of-job cleanup cannot find + processes of the namespace. They are killed with pid 1 instead (see + above). +- **CI coverage when a scan fails.** Isolated steps run last in their job + (see below). When the container scan fails, the Dev environment job stops + before `tests/devenv/unit.sh`, so the allocator unit tests do not run in + that CI run. + +#### CI + +The isolated steps are the last steps of each required job: + +- **Repository guard:** the trusted guard scripts, ShellCheck and + actionlint run first; then the guard regression suites and + `scripts/ci/tests/no-docker.test.sh`. +- **Dev environment:** `tests/devenv/unit.sh` runs after the container + scan. + +After them, only the checkout action's post-job step runs. During the +isolated steps on Linux it reads nothing that the code under test could +have written: the workspace and `$HOME` are read-only, and `/tmp` is +private. + +`scripts/ci/tests/ci-isolation-order.py` checks the order statically: + +- after the first isolated step of a job, every following step is + isolated. A step is isolated when a script it runs (directly or through + a `make` target) calls `no-docker.sh`; +- the required jobs exist; +- no job or step has an `if:` condition; +- every other file that mentions `no-docker.sh` has the recognised re-exec + stanza. + +#### Regression tests + +`scripts/ci/tests/no-docker.test.sh` runs a mutated `bin/devenv` whose +commands reach Docker in more than 20 ways, both as `DEVENV` and edited +into the file. The mutant only reads (`docker version`, `GET /version`); +its text is pinned by sha256 and checked before each use. Its target is a +canary engine of the test, which answers like Docker and records every +request. + +- **T-03 and T-04:** on macOS, `unit.sh` runs on the host as a developer + would run it. The mutant must run to its end without a request reaching + the canary. On Linux, `unit.sh` runs inside the isolation, where the + canary is a reachable engine, so it must refuse. +- **T-05 (control):** a copy of `unit.sh` without the re-execution, behind + an outer guard. The mutant must reach the canary. +- **T-14:** the helpers that the test runs outside any isolation, and the + mutant's text, are kept outside every directory that code under test can + write. Their sha256 is checked immediately before each such run. The + case also checks that a changed copy is detected and not run. + +The other cases run plain commands that only print, connect, write markers +in the test's directories or send signal 0 or TERM to the test's own +`sleep`: + +- **T-00:** the read-only lint of the mutant. +- **T-01 and T-02:** `--check`, and a claimed but unproven isolation. +- **T-07:** loopback ports that were already listening are unreachable, + while ports opened inside are reachable. +- **T-06 and T-08 to T-15, macOS only:** + - **T-06 and T-11:** the write limits; + - **T-08:** the complete port list; + - **T-09 and T-12:** the `TMPDIR` allowlist; + - **T-10:** preferences and `TIOCSTI`; + - **T-13:** a canary outside cannot be signalled (EPERM) and survives, + while the command's own child and process group can be signalled; + - **T-14:** integrity of the helpers; + - **T-15:** `--work`, accepted and refused cases. +- **T-16 to T-18, Linux only:** + - **T-16:** a canary inside sees only the namespace's processes, cannot + read `/proc/1/root` and cannot connect to any engine, session or host + socket. It gets EROFS in `$HOME`, the workspace, `$RUNNER_TEMP` and the + tool cache, and sees a private `/tmp` and `/dev/shm`, read-only mounts, + masked cgroups, private IPC and loopback only; + - **T-17:** a `setsid` daemon does not outlive the command, neither when + the command ends nor when `unshare` is killed; + - **T-18:** uid 0 (a real `sudo` run), missing or other runner variables, + a workspace outside `$HOME` and a caller without a `Runner.Worker` + ancestor are refused. +- **T-19:** the static CI order check, on both platforms. + +The macOS cases run only on a developer's Mac: there is no macOS CI job. + +#### Mutation testing - Run a mutated copy of `bin/devenv` or of a guard only through these - suites. -- Use payloads that only read or only print. Never use a destructive Docker - command (prune, forced removal, `down -v`, `--rmi`), not even in isolation. + suites, on a macOS developer machine or in CI. On Linux outside CI the + suites refuse to run. +- Until a CI run on the GitHub-hosted runner has shown the Linux cases + T-16 to T-18 passing, use only payloads that read or print, on both + platforms. +- Never use a destructive Docker command (prune, forced removal, `down -v`, + `--rmi`), not even in isolation. - `tests/devenv/integration.sh` and the `tests/app/` suites use the real Docker daemon on purpose, so never run them on a mutated copy. `integration.sh` refuses a `DEVENV` that differs from `bin/devenv`, but no From a0b1212886835a0eabf50cec662366691ae65a2d Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 09:25:37 +0100 Subject: [PATCH 14/77] fix(test): kill the Linux isolation when no-docker.sh is cancelled unshare runs as root and, with --fork, blocks SIGINT and SIGTERM, and pid 1 ignores signals from outside its namespace, so a cancelled or timed-out step left the namespace running. no-docker.sh now runs sudo in the background and, on SIGINT, SIGTERM or SIGHUP, sends SIGKILL through sudo to the unshare it started (a process named unshare below its sudo); --kill-child then kills pid 1 and everything inside. The command keeps the caller's stdin, which an asynchronous command would otherwise lose. The stop-commands start line also goes to stderr, and the duration of the socket listing is printed on stderr and as a notice in the CI log. --- scripts/ci/no-docker.sh | 62 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh index f2f7381..3cc2868 100755 --- a/scripts/ci/no-docker.sh +++ b/scripts/ci/no-docker.sh @@ -39,8 +39,9 @@ # read-only; the command runs as the calling user without # supplementary groups, capabilities or the means to regain them # (no_new_privs), in a session of its own, with an allowlisted -# environment, under a root pid 1 that waits for it; when it ends -# or the isolation is killed, every process left inside is killed. +# environment, under a root pid 1 that waits for it; when it ends, +# when no-docker.sh gets SIGINT, SIGTERM or SIGHUP (a cancelled +# step) or when unshare dies, every process left inside is killed. # Unix sockets found outside before the start must not accept a # connection inside. # In both, the variables that select an engine are removed and DOCKER_CONFIG @@ -108,6 +109,31 @@ owner_mode() { if [ "$os" = Darwin ]; then stat -f '%u %Lp' "$1"; else stat -c '%u %a' "$1"; fi } +# kill_isolation (Linux): SIGKILL the unshare that this script started: +# a process named unshare whose ancestors include $spid, the sudo of this +# script. Called from the INT/TERM/HUP trap. +# shellcheck disable=SC2329 +kill_isolation() { + local d p q st n + for d in /proc/[0-9]*; do + p=${d#/proc/} + [ "$(cat "$d/comm" 2>/dev/null)" = unshare ] || continue + q=$p + n=0 + while [ "$n" -lt 16 ]; do + st=$(cat "/proc/$q/stat" 2>/dev/null) || break + read -r _ q _ <<<"${st##*) }" + case $q in "" | *[!0-9]*) break ;; esac + [ "$q" -gt 1 ] || break + if [ "$q" = "$spid" ]; then + sudo -n kill -KILL "$p" 2>/dev/null + break + fi + n=$((n + 1)) + done + done +} + # use_work DIR PARENT...: check the caller's --work DIR and use it as $work. use_work() { local d=$1 canon real p inside=no @@ -294,7 +320,10 @@ case $os in # Every Unix socket on the local file systems and every bound one of # the host's network namespace; inside, none may accept a connection. # A time budget that runs out refuses rather than truncating the list. - python3 -I "$probe" --sockets >"$work/sockets.list" || refuse "cannot list the Unix sockets" + timing=$(python3 -I "$probe" --sockets 2>&1 >"$work/sockets.list") || refuse "cannot list the Unix sockets: $timing" + # Its duration, in the log of every run (the budget is 180 s). + printf '%s\n' "$timing" >&2 + printf '::notice title=no-docker::%s\n' "${timing#no-docker: }" envargs+=("NO_DOCKER_SOCKETS=/run/no-docker/sockets.list" "NO_DOCKER_HOST_NS=${host_ns# }" "NO_DOCKER_HOST_DEVS=${host_devs# }" "NO_DOCKER_PROTECTED=$protected") # pid 1 of the new pid namespace is this root shell. It does not exec: @@ -327,9 +356,13 @@ case $os in mkdir -p "$work" && mount --move /run/no-docker/work "$work" || die "cannot move $work into the private temporary directory" python3 -I "$probe" --seal "$work" || die "cannot make the other mounts read-only" cd "$cwd" || die "the working directory $cwd is not visible in the isolation" + # An asynchronous command gets /dev/null as stdin unless it is + # given one explicitly: that of the caller, through fd 3. + exec 3<&0 setpriv --reuid "$uid" --regid "$gid" --clear-groups --no-new-privs --inh-caps=-all --bounding-set=-all -- \ - setsid --wait /usr/bin/env -i "$@" & + setsid --wait /usr/bin/env -i "$@" <&3 3<&- & child=$! + exec 3<&- wait "$child" exit $? ' @@ -338,10 +371,27 @@ case $os in token=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') [ "${#token}" -eq 32 ] || refuse "cannot make a random token" printf '::stop-commands::%s\n' "$token" + printf '::stop-commands::%s\n' "$token" >&2 + # Cancellation: unshare runs as root and, with --fork, blocks SIGINT and + # SIGTERM; pid 1 ignores signals from outside its namespace. So a signal + # to this script (a cancelled or timed-out step) is turned into SIGKILL + # of the unshare started below, found as a descendant of its sudo; + # --kill-child then kills pid 1, and with it every process inside. + exec 3<&0 sudo -n unshare --net --mount --pid --fork --kill-child --mount-proc --ipc --propagation private -- \ /bin/sh -c "$init" no-docker-init "$uid" "$(id -g)" "$work" "$probe" "$cwd" \ - "${envargs[@]}" "$BASH" "$here/no-docker.sh" --inner "$@" - rc=$? + "${envargs[@]}" "$BASH" "$here/no-docker.sh" --inner "$@" <&3 3<&- & + spid=$! + exec 3<&- + interrupted="" + trap 'interrupted=yes; kill_isolation' INT TERM HUP + while :; do + wait "$spid" + rc=$? + [ -n "$interrupted" ] || break + interrupted="" + done + trap - INT TERM HUP printf '::%s::\n' "$token" ;; esac From e47d09ee6d1f1d9a853b748bdf4c4c6151e0f17e Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 09:25:37 +0100 Subject: [PATCH 15/77] test(ci): make T-17 prove its canary ran and cover cancellation by TERM T-17's canary is a shell kept alive ("sleep 60; :") so the process with the marker is the one that must die, and the command confirms it runs before returning; the cancellation case sends TERM to no-docker.sh, as a cancelled step does. The macOS guard used for T-05 and HARNESS= gets the write policy and denies of no-docker.sh. Files that code under test could write are read only when they are regular files. T-19 counts a step as isolated only when all its commands are no-docker.sh, its test harness or scripts with the re-exec stanza, and fails a step that mixes them. --- scripts/ci/tests/ci-isolation-order.py | 59 ++++++++++------- scripts/ci/tests/no-docker.test.sh | 91 ++++++++++++++++---------- 2 files changed, 93 insertions(+), 57 deletions(-) diff --git a/scripts/ci/tests/ci-isolation-order.py b/scripts/ci/tests/ci-isolation-order.py index 0daa180..e1030bd 100644 --- a/scripts/ci/tests/ci-isolation-order.py +++ b/scripts/ci/tests/ci-isolation-order.py @@ -2,9 +2,12 @@ Usage: ci-isolation-order.py REPO_ROOT [WORKFLOW] (default .github/workflows/ci.yml) -A step is isolated when a script it runs (named in its `run:` text, or in -the recipe of a `make` target it runs) calls scripts/ci/no-docker.sh: it -runs code under test, which may leave files or processes for later steps. +A step is isolated when every command it runs (in its `run:` text, or in +the recipe of a `make` target it runs) is scripts/ci/no-docker.sh, its +test harness scripts/ci/tests/no-docker.test.sh, or a script with the +recognised re-exec stanza: it runs code under test, which may leave files +or processes for later steps. A step that mixes such commands with others +fails. Code under test cannot write anything on the Linux runner's file systems except private temporary directories, but the order is the second line: after the first isolated step of a job, only isolated steps may follow. @@ -31,6 +34,9 @@ "scripts/ci/tests/ci-isolation-order.py", ".github/workflows/ci.yml"} SCRIPT = re.compile(r"(?/dev/null 2>&1; then sha256sum; else shasum -a 256; fi; } <"$1" | cut -c1-64 } +# Files that code under test could write are read only when they are +# regular files, so a planted FIFO or link cannot hang or redirect a read. +regular() { [ -f "$1" ] && [ ! -L "$1" ]; } +count_in() { # pattern file: number of matching lines, 0 if not a regular file + local n=0 + regular "$2" && n=$(grep -c -- "$1" <"$2" 2>/dev/null) + echo "${n:-0}" +} + new_work() { # a fresh, empty directory for --work mktemp -d "$vt/w.XXXXXX" } @@ -335,19 +344,34 @@ run_case() { case $os-$mode in Darwin-bare) bare "$helpers" "$data" "$canary" "${cmd[@]}" ;; Darwin-guard) + # The write policy of no-docker.sh: only the case's directories, the + # /dev nodes of its profile and bash 3.2's here-documents; plus its + # other denies and the ports listening now. Only the canary socket + # is reachable. verify_helpers "$helpers" || exit 2 + held="" + for port in $(python3 -I "$root/scripts/ci/no-docker-probe.py" --listening) ; do + case $port in *[!0-9]* | "") echo "HARNESS ERROR: unexpected port $port" >&2; exit 2 ;; esac + held="$held (remote ip \"localhost:$port\")" + done sandbox-exec -p "(version 1) (allow default) (deny network-outbound (remote unix-socket)) (allow network-outbound (remote unix-socket (path-literal \"$canary\"))) (deny network-outbound (remote ip)) (allow network-outbound (remote ip \"localhost:*\")) -(deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")) -(deny file-write* (subpath \"$home\") (subpath \"$ro\")) +(deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")$held) +(deny file-write*) +(allow file-write* (subpath \"$data\") (subpath \"$w\") (literal \"/dev/null\") (literal \"/dev/zero\") (literal \"/dev/tty\") (literal \"/dev/dtracehelper\") (regex #\"^/dev/fd/[0-9]+\$\") (literal \"/private/tmp\") (regex #\"^/private/tmp/sh-thd[^/]*\$\")) +(deny user-preference-write) +(deny file-ioctl (ioctl-command #x80017472)) (deny signal) (allow signal (target same-sandbox)) (deny process-exec (literal \"/usr/bin/open\") (literal \"/usr/bin/osascript\") (literal \"/bin/launchctl\") (literal \"/usr/bin/ssh\")) -(deny mach-lookup (global-name \"com.apple.coreservices.launchservicesd\") (global-name \"com.apple.coreservices.appleevents\") (global-name-regex #\"^com\\.apple\\.lsd\\.\"))" \ +(deny job-creation) +(deny lsopen) +(deny appleevent-send) +(deny mach-lookup (global-name \"com.apple.coreservices.launchservicesd\") (global-name \"com.apple.coreservices.appleevents\") (global-name-regex #\"^com\\.apple\\.lsd\\.\") (global-name \"com.apple.xpc.smd\") (global-name-regex #\"^com\\.apple\\.backgroundtaskmanagement\") (global-name-regex #\"^com\\.apple\\.coreservices\\.sharedfilelistd\") (global-name-regex #\"^com\\.docker\\.\") (global-name-regex #\"^com\\.electron\\.dockerdesktop\\.\"))" \ "$BASH" "$helpers/drive.sh" "$helpers/canary.py" "$data" "$canary" "${cmd[@]}" ;; *) @@ -358,8 +382,7 @@ run_case() { RC=$? STARTED=1 [ -e "$data/drive-started" ] || STARTED=0 - REQ=$(grep -c . "$data/canary.log" 2>/dev/null) - REQ=${REQ:-0} + REQ=$(count_in . "$data/canary.log") } # The harness without the containment: the lines that re-execute it under @@ -789,43 +812,46 @@ with open("/proc/net/unix", "rb") as f: rm -f "$t16_tmp" "$t16_shm" "$t16_in" # --- T-17 no process survives the isolated command: a `setsid -f` canary - # with a unique marker, after a normal end and after the isolation itself - # (unshare) is killed. Checked from outside, after no-docker.sh returned. - # On 38b5ebd: no pid namespace and no --kill-child, the canary survives; - # and there is no unshare process to kill (it exec'd), which fails too. + # with a unique marker, after a normal end and after the isolation was + # cancelled (TERM to no-docker.sh, as a cancelled CI step does). The + # canary is a shell kept alive by "sleep 60; :", so the process with the + # marker in its command line is the one that must die; the command checks + # that it runs before it returns. Checked from outside, after + # no-docker.sh returned. + # On 38b5ebd: no pid namespace and no --kill-child, so the canary + # survives; and TERM to no-docker.sh left sudo, unshare and the command + # running. t17m=t17-$nonce - "$nd" /bin/sh -c 'setsid -f /bin/sh -c "sleep 60" "$1-in" /dev/null 2>&1; exit 0' _ "$t17m" >/dev/null 2>&1 - r=$? + t17=$("$nd" /bin/sh -c 'setsid -f /bin/sh -c "sleep 60; :" "$1-in" /dev/null 2>&1 +for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do + if pgrep -f -- "$1-in" >/dev/null; then echo canary-started; exit 0; fi + sleep 0.1 +done +echo canary-missing' _ "$t17m" 2>/dev/null | grep -v '^::') for _ in 1 2 3 4 5 6 7 8 9 10; do pgrep -f -- "$t17m-in" >/dev/null || break; sleep 0.5; done pgrep -f -- "$t17m-in" >/dev/null left=$? pkill -TERM -f -- "$t17m-in" 2>/dev/null - [ "$r" -eq 0 ] && [ "$left" -ne 0 ] - result "T-17 a daemonised process does not outlive the isolated command" $? "(rc $r, survivor $([ "$left" -eq 0 ] && echo yes || echo no))" + [ "$t17" = canary-started ] && [ "$left" -ne 0 ] + result "T-17 a daemonised process does not outlive the isolated command" $? \ + "($t17, survivor $([ "$left" -eq 0 ] && echo yes || echo no))" t17c=t17c-$nonce - "$nd" /bin/sh -c 'setsid -f /bin/sh -c "sleep 60" "$1-in" /dev/null 2>&1; sleep 60' _ "$t17c" >/dev/null 2>&1 & + "$nd" /bin/sh -c 'setsid -f /bin/sh -c "sleep 60; :" "$1-in" /dev/null 2>&1; sleep 60; :' _ "$t17c" >/dev/null 2>&1 & t17pid=$! for _ in $(seq 1 240); do pgrep -f -- "$t17c-in" >/dev/null && break; sleep 0.5; done started=no pgrep -f -- "$t17c-in" >/dev/null && started=yes - upid="" - for p in $(pgrep -x unshare); do - q=$p - while [ -n "$q" ] && [ "$q" -gt 1 ]; do - q=$(ps -o ppid= -p "$q" 2>/dev/null | tr -d ' ') - if [ "$q" = "$t17pid" ]; then upid=$p; break 2; fi - done - done - [ -z "$upid" ] || sudo -n kill -KILL "$upid" + kill -TERM "$t17pid" 2>/dev/null wait "$t17pid" 2>/dev/null + r=$? for _ in 1 2 3 4 5 6 7 8 9 10; do pgrep -f -- "$t17c-in" >/dev/null || break; sleep 0.5; done pgrep -f -- "$t17c-in" >/dev/null left=$? pkill -TERM -f -- "$t17c" 2>/dev/null - [ "$started" = yes ] && [ -n "$upid" ] && [ "$left" -ne 0 ] - result "T-17 killing the isolation (unshare) kills every process inside it" $? \ - "(canary started $started, unshare ${upid:-not found}, survivor $([ "$left" -eq 0 ] && echo yes || echo no))" + [ "$started" = yes ] && [ "$left" -ne 0 ] + result "T-17 TERM to no-docker.sh (a cancelled step) kills every process inside the isolation" $? \ + "(canary started $started, rc $r, survivor $([ "$left" -eq 0 ] && echo yes || echo no))" # --- T-18 unsupported use is refused: uid 0 (a real sudo run) and a Linux # that is not a GitHub-hosted runner. The command is `true`. @@ -870,10 +896,8 @@ check_safe() { # name # Refused (125, the mutant never ran) or run isolated (97, every case ran # and none reached an engine); either way the canary engine got nothing. local cases answered - cases=$(grep -c ' rc=' "$LOG" 2>/dev/null) - cases=${cases:-0} - answered=$(grep -c 'canary-engine' "$LOG" 2>/dev/null) - answered=${answered:-0} + cases=$(count_in ' rc=' "$LOG") + answered=$(count_in 'canary-engine' "$LOG") [ "$REQ" -eq 0 ] && [ "$answered" -eq 0 ] result "$1: no request reached an engine" $? "(canary requests $REQ, answered cases $answered, rc $RC)" [ "$STARTED" -eq 1 ] @@ -888,7 +912,7 @@ check_safe() { # name grep -q '^no-docker: refused: inside no-docker.sh, but the isolation check fails' "$OUT" result "$1: the harness refused before the mutant ran (an engine is reachable)" $? "(rc $RC, $cases cases)" fi - ! grep -E '^(real-socket-cli|real-socket-curl) rc=0 ' "$LOG" >/dev/null 2>&1 + [ "$(count_in '^real-socket-c[a-z]* rc=0 ' "$LOG")" -eq 0 ] result "$1: the real Docker socket paths were not reached" $? [ ! -e "$home_marker" ] result "$1: nothing written under \$HOME" $? @@ -904,8 +928,7 @@ check_safe "T-04 mutant edited into bin/devenv" # --- T-05 control: the same check catches a harness without containment run_case t05 "$ro/unit-uncontained.sh" inplace guard shown+=("t05 $LOG $OUT") -answered=$(grep -c 'canary-engine' "$LOG" 2>/dev/null) -answered=${answered:-0} +answered=$(count_in 'canary-engine' "$LOG") [ "$STARTED" -eq 1 ] && [ "$REQ" -gt 0 ] && [ "$answered" -gt 0 ] && [ "$RC" -eq 97 ] result "T-05 control: without the containment the mutant's commands reach the canary engine" $? "(canary requests $REQ, answered cases $answered, rc $RC)" @@ -913,7 +936,7 @@ echo for s in "${shown[@]}"; do read -r f log out <<<"$s" printf 'INFO %s cases:\n' "$f" - sed 's/^/INFO /' "$log" 2>/dev/null + ! regular "$log" || sed 's/^/INFO /' <"$log" 2>/dev/null printf 'INFO %s harness output (last lines):\n' "$f" tail -n 8 "$out" 2>/dev/null | sed 's/^/INFO /' done From de1d42ffda370dc6d4592eaa884a08e5b7954bdf Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 09:25:37 +0100 Subject: [PATCH 16/77] docs(dev-environment): describe cancellation, the here-document residual and the stricter CI check The Linux isolation is killed when no-docker.sh is cancelled; SIGKILL of no-docker.sh itself remains a residual until the VM is discarded. bash 3.2 here-documents in /tmp, measured to ignore $TMPDIR, stay allowed and are a documented macOS residual. Also the stop-commands framing on both streams, the listing duration in the CI log, T-17, T-05's guard and the definition of an isolated step. --- docs/dev-environment.md | 59 ++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/docs/dev-environment.md b/docs/dev-environment.md index 3e175b1..71e38ba 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -528,7 +528,8 @@ local file systems (`find -xdev -type s` per mount) and every bound pathname socket of the host's network namespace (`/proc/net/unix`, which is what `ss -xl` shows). The list has a 180-second budget; when it runs out, the run is refused instead of using a partial list. The time it took is -printed with each run. Inside, the list is a read-only file. +printed with each run, on stderr and as a `::notice::` in the CI log. +Inside, the list is a read-only file. The command then runs under `sudo -n unshare --net --mount --pid --fork --kill-child --mount-proc --ipc` @@ -556,9 +557,19 @@ does not exec: runner file-command or token variable reaches the command. 5. It waits for the command, reaping orphans meanwhile, and exits with the command's status. When pid 1 exits, the kernel kills every process left - in the namespace, including daemonised ones. If `unshare` itself is - killed (a cancelled job or a timeout), `--kill-child` kills pid 1. pid 1 - makes no credential change, so that signal stays armed. + in the namespace, including daemonised ones. If `unshare` itself dies, + `--kill-child` kills pid 1. pid 1 makes no credential change, so that + signal stays armed. + +Cancellation: `unshare` runs as root, and with `--fork` it blocks SIGINT and +SIGTERM; pid 1 ignores signals from outside its namespace. So when +`no-docker.sh` gets SIGINT, SIGTERM or SIGHUP (a cancelled or timed-out +step), it sends SIGKILL, through `sudo`, to the `unshare` it started: the +process named `unshare` whose ancestors include the `sudo` of this run. +Through `--kill-child`, that kills pid 1 and every process inside. If +`no-docker.sh` itself is killed with SIGKILL first, nothing can forward +the cancellation: the namespace then runs until the runner's VM is +discarded (see the residual risks). The probe checks from inside: @@ -588,7 +599,8 @@ The probe checks from inside: The output between the start and the end of the isolated command is framed with `::stop-commands::`, so the command cannot issue -workflow commands. The token is never passed inside. +workflow commands. The start line is printed on both stdout and stderr; the +token is never passed inside. #### Run directory (`--work DIR`) @@ -626,6 +638,17 @@ them. services that register login items and background tasks are denied. A re-signed copy of `launchctl` loses its entitlements. Whether such a copy can still `setenv` in the user's launchd domain was not measured. +- **Here-document files in `/tmp` (macOS).** bash 3.2, which is also + `/bin/sh`, writes every here-document to `/tmp/sh-thd*` whatever + `$TMPDIR` is. It creates the file, closes it, opens it again by name and + then removes it. Measured: without the `sh-thd*` allowance, here-documents + in the sandbox fail ("cannot create temp file for here document"), also + with `$TMPDIR` in the run's directory. So the allowance stays. Code in + the sandbox can therefore remove and recreate an `sh-thd*` file of a + bash 3.2 that runs outside the sandbox, between that bash's close and + reopen. That injects shell input into the outside script. The window is + short, but a leftover process (above) can wait for it. SBPL cannot limit + the rule to files the sandbox created itself. - **Terminals and the pasteboard (macOS).** Writes to other terminals' devices are denied. The pasteboard (`pbcopy`) is not; using it needs a person to paste. @@ -643,8 +666,11 @@ them. but there is no network to use them on. - **Orphan cleanup after a kill (Linux).** `env -i` drops `RUNNER_TRACKING_ID`, so the runner's end-of-job cleanup cannot find - processes of the namespace. They are killed with pid 1 instead (see - above). + processes of the namespace. They are killed with pid 1 instead, on a + normal end and when `no-docker.sh` gets SIGINT, SIGTERM or SIGHUP (see + above). If `no-docker.sh` is killed with SIGKILL before it can forward + a cancellation, the namespace keeps running, with every limit above, + until the runner's VM is discarded. - **CI coverage when a scan fails.** Isolated steps run last in their job (see below). When the container scan fails, the Dev environment job stops before `tests/devenv/unit.sh`, so the allocator unit tests do not run in @@ -668,8 +694,10 @@ private. `scripts/ci/tests/ci-isolation-order.py` checks the order statically: - after the first isolated step of a job, every following step is - isolated. A step is isolated when a script it runs (directly or through - a `make` target) calls `no-docker.sh`; + isolated. A step is isolated when every command it runs (directly or + through a `make` target) is `no-docker.sh`, its test harness + `no-docker.test.sh`, or a script with the re-exec stanza. A step that + mixes such commands with others fails; - the required jobs exist; - no job or step has an `if:` condition; - every other file that mentions `no-docker.sh` has the recognised re-exec @@ -689,7 +717,13 @@ request. the canary. On Linux, `unit.sh` runs inside the isolation, where the canary is a reachable engine, so it must refuse. - **T-05 (control):** a copy of `unit.sh` without the re-execution, behind - an outer guard. The mutant must reach the canary. + an outer guard. The mutant must reach the canary. On macOS, that guard + (also used for every case with `HARNESS=`) has the same write policy and + denies as `no-docker.sh`. Only the case's own directories are writable, + and only the canary's socket is reachable. +- Files that code under test could write are read only when they are + regular files, so a planted FIFO or link cannot hang or redirect the + test. - **T-14:** the helpers that the test runs outside any isolation, and the mutant's text, are kept outside every directory that code under test can write. Their sha256 is checked immediately before each such run. The @@ -718,8 +752,9 @@ in the test's directories or send signal 0 or TERM to the test's own socket. It gets EROFS in `$HOME`, the workspace, `$RUNNER_TEMP` and the tool cache, and sees a private `/tmp` and `/dev/shm`, read-only mounts, masked cgroups, private IPC and loopback only; - - **T-17:** a `setsid` daemon does not outlive the command, neither when - the command ends nor when `unshare` is killed; + - **T-17:** a `setsid` daemon, checked to be running before the command + returns, does not outlive the command, neither when the command ends + nor when `no-docker.sh` gets SIGTERM (a cancelled step); - **T-18:** uid 0 (a real `sudo` run), missing or other runner variables, a workspace outside `$HOME` and a caller without a `Runner.Worker` ancestor are refused. From dfb6ee4fa7efb327b08803ffb3de81580d493a73 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 09:34:48 +0100 Subject: [PATCH 17/77] docs(dev-environment): state that a cancelled CI step does not reach the isolation's trap The INT/TERM/HUP trap of no-docker.sh handles signals delivered to the script itself, such as Ctrl-C or a caller forwarding the signal. A cancelled or timed-out GitHub step signals only the step's shell and later SIGKILLs the process tree, so it does not reach the trap; the remaining namespace keeps read-only mounts and no network until the VM is discarded, an accepted residual. Comments, documentation and the T-17 description only; no logic changes. --- docs/dev-environment.md | 38 +++++++++++++++++++----------- scripts/ci/no-docker.sh | 22 +++++++++++------ scripts/ci/tests/no-docker.test.sh | 7 +++--- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/docs/dev-environment.md b/docs/dev-environment.md index 71e38ba..93fdf78 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -561,15 +561,19 @@ does not exec: `--kill-child` kills pid 1. pid 1 makes no credential change, so that signal stays armed. -Cancellation: `unshare` runs as root, and with `--fork` it blocks SIGINT and -SIGTERM; pid 1 ignores signals from outside its namespace. So when -`no-docker.sh` gets SIGINT, SIGTERM or SIGHUP (a cancelled or timed-out -step), it sends SIGKILL, through `sudo`, to the `unshare` it started: the -process named `unshare` whose ancestors include the `sudo` of this run. -Through `--kill-child`, that kills pid 1 and every process inside. If -`no-docker.sh` itself is killed with SIGKILL first, nothing can forward -the cancellation: the namespace then runs until the runner's VM is -discarded (see the residual risks). +Interruption: `unshare` runs as root, and with `--fork` it blocks SIGINT +and SIGTERM; pid 1 ignores signals from outside its namespace. So when +SIGINT, SIGTERM or SIGHUP is delivered to `no-docker.sh` itself (Ctrl-C in +a terminal, or a caller that forwards the signal), it sends SIGKILL, +through `sudo`, to the `unshare` it started: the process named `unshare` +whose ancestors include the `sudo` of this run. Through `--kill-child`, +that kills pid 1 and every process inside. + +A cancelled or timed-out GitHub step does not reach this. The runner +signals only the step's shell, and later sends SIGKILL to the process +tree, so `no-docker.sh` gets no signal it can handle. The namespace then +keeps running, with read-only mounts and no network, until the runner's +VM is discarded (see the residual risks). The probe checks from inside: @@ -667,10 +671,14 @@ them. - **Orphan cleanup after a kill (Linux).** `env -i` drops `RUNNER_TRACKING_ID`, so the runner's end-of-job cleanup cannot find processes of the namespace. They are killed with pid 1 instead, on a - normal end and when `no-docker.sh` gets SIGINT, SIGTERM or SIGHUP (see - above). If `no-docker.sh` is killed with SIGKILL before it can forward - a cancellation, the namespace keeps running, with every limit above, - until the runner's VM is discarded. + normal end and when SIGINT, SIGTERM or SIGHUP is delivered to + `no-docker.sh` itself (see above). +- **Cancelled or timed-out CI steps (Linux).** GitHub signals only the + step's shell and later sends SIGKILL to the process tree. Neither + reaches the trap in `no-docker.sh`, so the namespace and anything left + in it keep running until the runner's VM is discarded. They still have + read-only mounts, no network, no capabilities and no view of other + processes. Only the checkout's post-job step follows. - **CI coverage when a scan fails.** Isolated steps run last in their job (see below). When the container scan fails, the Dev environment job stops before `tests/devenv/unit.sh`, so the allocator unit tests do not run in @@ -754,7 +762,9 @@ in the test's directories or send signal 0 or TERM to the test's own masked cgroups, private IPC and loopback only; - **T-17:** a `setsid` daemon, checked to be running before the command returns, does not outlive the command, neither when the command ends - nor when `no-docker.sh` gets SIGTERM (a cancelled step); + nor when SIGTERM is delivered to `no-docker.sh` itself. A cancelled + GitHub step does not signal `no-docker.sh`, so that case is not + covered; - **T-18:** uid 0 (a real `sudo` run), missing or other runner variables, a workspace outside `$HOME` and a caller without a `Runner.Worker` ancestor are refused. diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh index 3cc2868..dab110c 100755 --- a/scripts/ci/no-docker.sh +++ b/scripts/ci/no-docker.sh @@ -40,8 +40,12 @@ # supplementary groups, capabilities or the means to regain them # (no_new_privs), in a session of its own, with an allowlisted # environment, under a root pid 1 that waits for it; when it ends, -# when no-docker.sh gets SIGINT, SIGTERM or SIGHUP (a cancelled -# step) or when unshare dies, every process left inside is killed. +# when no-docker.sh itself gets SIGINT, SIGTERM or SIGHUP (such as +# Ctrl-C, or a caller forwarding the signal) or when unshare dies, +# every process left inside is killed. A cancelled or timed-out +# GitHub step does not signal no-docker.sh: its namespace keeps +# running, read-only and without network, until the VM is +# discarded (an accepted residual). # Unix sockets found outside before the start must not accept a # connection inside. # In both, the variables that select an engine are removed and DOCKER_CONFIG @@ -372,11 +376,15 @@ case $os in [ "${#token}" -eq 32 ] || refuse "cannot make a random token" printf '::stop-commands::%s\n' "$token" printf '::stop-commands::%s\n' "$token" >&2 - # Cancellation: unshare runs as root and, with --fork, blocks SIGINT and - # SIGTERM; pid 1 ignores signals from outside its namespace. So a signal - # to this script (a cancelled or timed-out step) is turned into SIGKILL - # of the unshare started below, found as a descendant of its sudo; - # --kill-child then kills pid 1, and with it every process inside. + # Interruption: unshare runs as root and, with --fork, blocks SIGINT and + # SIGTERM; pid 1 ignores signals from outside its namespace. So SIGINT, + # SIGTERM or SIGHUP delivered to this script itself (Ctrl-C, or a caller + # forwarding the signal) is turned into SIGKILL of the unshare started + # below, found as a descendant of its sudo; --kill-child then kills + # pid 1, and with it every process inside. A cancelled or timed-out + # GitHub step signals only the step's shell and later SIGKILLs the + # process tree, so it does not reach this trap: the namespace keeps + # running, read-only and without network, until the VM is discarded. exec 3<&0 sudo -n unshare --net --mount --pid --fork --kill-child --mount-proc --ipc --propagation private -- \ /bin/sh -c "$init" no-docker-init "$uid" "$(id -g)" "$work" "$probe" "$cwd" \ diff --git a/scripts/ci/tests/no-docker.test.sh b/scripts/ci/tests/no-docker.test.sh index 5df1b46..59a2fe7 100755 --- a/scripts/ci/tests/no-docker.test.sh +++ b/scripts/ci/tests/no-docker.test.sh @@ -812,8 +812,9 @@ with open("/proc/net/unix", "rb") as f: rm -f "$t16_tmp" "$t16_shm" "$t16_in" # --- T-17 no process survives the isolated command: a `setsid -f` canary - # with a unique marker, after a normal end and after the isolation was - # cancelled (TERM to no-docker.sh, as a cancelled CI step does). The + # with a unique marker, after a normal end and after TERM delivered to + # no-docker.sh itself (as by Ctrl-C or a caller forwarding the signal; a + # cancelled GitHub step does not signal no-docker.sh and is not covered). The # canary is a shell kept alive by "sleep 60; :", so the process with the # marker in its command line is the one that must die; the command checks # that it runs before it returns. Checked from outside, after @@ -850,7 +851,7 @@ echo canary-missing' _ "$t17m" 2>/dev/null | grep -v '^::') left=$? pkill -TERM -f -- "$t17c" 2>/dev/null [ "$started" = yes ] && [ "$left" -ne 0 ] - result "T-17 TERM to no-docker.sh (a cancelled step) kills every process inside the isolation" $? \ + result "T-17 TERM delivered to no-docker.sh itself kills every process inside the isolation" $? \ "(canary started $started, rc $r, survivor $([ "$left" -eq 0 ] && echo yes || echo no))" # --- T-18 unsupported use is refused: uid 0 (a real sudo run) and a Linux From 0cb7f376f618ba26a4947ae2d63bbc24cfd90035 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 17:08:43 +0100 Subject: [PATCH 18/77] fix(devenv): print usage and read container labels without here-documents bash 3.2 keeps here-documents and here-strings in a temporary file in /var/tmp, or in /tmp where /var/tmp is not writable, as inside the macOS sandbox of scripts/ci/no-docker.sh, which no longer allows either. usage prints its lines with printf (same output), require_owned_project reads the container list through a process substitution (the loop still runs in the current shell, so its refusals still exit) and proc_start splits /proc//stat the same way without a here-string. --- bin/devenv | 60 ++++++++++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/bin/devenv b/bin/devenv index a943548..aae96cd 100755 --- a/bin/devenv +++ b/bin/devenv @@ -68,33 +68,33 @@ show() { } usage() { - cat <<'EOF' -Usage: bin/devenv - - up Start the stack and print its URL (port chosen automatically) - down Stop and remove this checkout's containers (volumes are kept) - restart down + up, preferring the saved port - status Show container state and the saved port (exit 1 if not running) - url Print the verified URL (exit 1 if not running) - ports Show candidate ports and who uses them (read-only) - logs Show recent logs (SERVICE=, TAIL=200, FOLLOW=1) - doctor Check prerequisites (read-only) - upgrade-roles - Move a database volume created before role separation to the - three roles of ADR-018, after a verified backup (explicit only) - rotate-db-password - New password for ROLE=app or ROLE=owner (development only; - recreates the containers that use it). Run again after an - interruption to finish the rotation - rotate-db-password-abort - Undo an unfinished rotation of ROLE=app or ROLE=owner: the - database gets the current password back - -Configuration: FLOWCRM_STACK (app or fixture), FLOWCRM_PORT_DEFAULT (8080), -FLOWCRM_PORT_RANGE (8080-8099), FLOWCRM_HTTP_PORT (explicit port, no -fallback), FLOWCRM_UP_TIMEOUT (600), FLOWCRM_WAIT_TIMEOUT (60), -FLOWCRM_LOCK_WAIT (120). See docs/dev-environment.md. -EOF + # One argument per line: no here-document (bash 3.2 keeps those in /tmp). + printf '%s\n' \ + 'Usage: bin/devenv ' \ + '' \ + ' up Start the stack and print its URL (port chosen automatically)' \ + ' down Stop and remove this checkout'\''s containers (volumes are kept)' \ + ' restart down + up, preferring the saved port' \ + ' status Show container state and the saved port (exit 1 if not running)' \ + ' url Print the verified URL (exit 1 if not running)' \ + ' ports Show candidate ports and who uses them (read-only)' \ + ' logs Show recent logs (SERVICE=, TAIL=200, FOLLOW=1)' \ + ' doctor Check prerequisites (read-only)' \ + ' upgrade-roles' \ + ' Move a database volume created before role separation to the' \ + ' three roles of ADR-018, after a verified backup (explicit only)' \ + ' rotate-db-password' \ + ' New password for ROLE=app or ROLE=owner (development only;' \ + ' recreates the containers that use it). Run again after an' \ + ' interruption to finish the rotation' \ + ' rotate-db-password-abort' \ + ' Undo an unfinished rotation of ROLE=app or ROLE=owner: the' \ + ' database gets the current password back' \ + '' \ + 'Configuration: FLOWCRM_STACK (app or fixture), FLOWCRM_PORT_DEFAULT (8080),' \ + 'FLOWCRM_PORT_RANGE (8080-8099), FLOWCRM_HTTP_PORT (explicit port, no' \ + 'fallback), FLOWCRM_UP_TIMEOUT (600), FLOWCRM_WAIT_TIMEOUT (60),' \ + 'FLOWCRM_LOCK_WAIT (120). See docs/dev-environment.md.' } # --- validation ------------------------------------------------------------- @@ -393,7 +393,7 @@ proc_start() { local stat boot f if [ -r /proc/sys/kernel/random/boot_id ] && stat=$(cat "/proc/$1/stat" 2>/dev/null); then boot=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null) - read -r -a f <<<"${stat##*) }" + read -r -a f < <(printf '%s\n' "${stat##*) }") if [ -n "$boot" ] && [[ ${f[19]-} =~ ^[0-9]+$ ]]; then printf 'linux:%s:%s\n' "$boot" "${f[19]}"; fi return 0 fi @@ -758,9 +758,7 @@ require_owned_project() { # recorded on each container tells them apart. [ "$files" = "$COMPOSE_FILE" ] || die $EX_SAFETY "project $PROJECT has containers of another stack (container $id from $(show "${files##*/}")); this command manages FLOWCRM_STACK=$STACK ($COMPOSE_FILE_NAME). Stop them first with: FLOWCRM_STACK=$(stack_of_file "$files") make down" - done < Date: Fri, 30 Oct 2026 17:08:43 +0100 Subject: [PATCH 19/77] test(devenv): write the unit suite's helper files without here-documents The suite runs inside the isolation, where bash 3.2 here-documents fail on macOS. The nine helper and fake files are written with printf; each file is byte for byte what the here-document produced. --- tests/devenv/unit.sh | 90 ++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 54 deletions(-) diff --git a/tests/devenv/unit.sh b/tests/devenv/unit.sh index cdd6182..c6e4ec9 100755 --- a/tests/devenv/unit.sh +++ b/tests/devenv/unit.sh @@ -280,30 +280,23 @@ result "U-09 no volume removal or global prune in bin/devenv" $r # bin/devenv, then runs MODE: "hold" waits for GO itself; the other modes # run a script from $work through run_op (recorded, own process group). # await.sh READY GO: create READY, then wait (at most 30 s) for GO. -cat >"$work/await.sh" <<'AWAIT' -: >"$1" +printf '%s\n' ': >"$1" i=0 -while [ ! -e "$2" ] && [ "$i" -lt 300 ]; do sleep 0.1; i=$((i + 1)); done -AWAIT +while [ ! -e "$2" ] && [ "$i" -lt 300 ]; do sleep 0.1; i=$((i + 1)); done' >"$work/await.sh" # tree.sh READY GO GCPID: like a Docker CLI with its Compose plugin, start a # descendant that waits for GO, then wait for it. -cat >"$work/tree.sh" <<'TREE' -sh "${0%/*}/await.sh" "$1.gc" "$2" & +printf '%s\n' 'sh "${0%/*}/await.sh" "$1.gc" "$2" & echo $! >"$3" i=0 while [ ! -e "$1.gc" ] && [ "$i" -lt 100 ]; do sleep 0.1; i=$((i + 1)); done : >"$1" -wait -TREE +wait' >"$work/tree.sh" # termchild.sh READY MARK: record a received SIGTERM in MARK, then exit. -cat >"$work/termchild.sh" <<'TERMCHILD' -trap ': >"$2"; exit 0' TERM +printf '%s\n' 'trap '\'': >"$2"; exit 0'\'' TERM : >"$1" i=0 -while [ "$i" -lt 300 ]; do sleep 0.1; i=$((i + 1)); done -TERMCHILD -cat >"$work/lockctl.sh" <<'LOCKCTL' -# lockctl.sh DEVENV RUNTIME_DIR LOCK_WAIT [hold|child|tree|term ARGS...] +while [ "$i" -lt 300 ]; do sleep 0.1; i=$((i + 1)); done' >"$work/termchild.sh" +printf '%s\n' '# lockctl.sh DEVENV RUNTIME_DIR LOCK_WAIT [hold|child|tree|term ARGS...] . "$1" SELF=$1 RUNTIME_DIR=$2 @@ -311,15 +304,14 @@ LOCK_DIR=$2/lock LOCK_WAIT=$3 work=${0%/*} trap on_exit EXIT -trap 'on_signal 143 TERM' TERM +trap '\''on_signal 143 TERM'\'' TERM acquire_lock test case ${4-} in hold) sh "$work/await.sh" "$5" "$6" ;; child) run_op sh "$work/await.sh" "$5" "$6" ;; tree) run_op sh "$work/tree.sh" "$5" "$6" "$7" ;; term) run_op sh "$work/termchild.sh" "$5" "$6" ;; -esac -LOCKCTL +esac' >"$work/lockctl.sh" # An array, not a function: env and & must start the lock owner itself. lockctl=("$BASH" "$work/lockctl.sh" "$devenv_script" "$RUNTIME_DIR") wait_file() { # file: bounded wait (10 s) for a readiness signal @@ -489,16 +481,14 @@ result "U-11 compose_op() refuses to run without the lock" $? "(rc $r: $out)" # try_edge() with a fake docker whose `compose up` prints a bind conflict and # then exits 1 or is killed by SIGKILL; `compose ps` reports no edge. mkdir -p "$work/fakeedge" -cat >"$work/fakeedge/docker" <<'FAKE' -#!/bin/sh +printf '%s\n' '#!/bin/sh case " $* " in *" up "*) - echo 'Error response from daemon: Bind for 127.0.0.1:8080 failed: port is already allocated' >&2 + echo '\''Error response from daemon: Bind for 127.0.0.1:8080 failed: port is already allocated'\'' >&2 [ "$FAKE_MODE" = killed ] && kill -KILL $$ exit 1 ;; esac -exit 0 -FAKE +exit 0' >"$work/fakeedge/docker" chmod +x "$work/fakeedge/docker" # shellcheck disable=SC2030,SC2031 # PATH and globals change only inside the subshell, on purpose edge_rc() { # mode: try_edge result with the fake docker @@ -700,15 +690,14 @@ result "U-15 the URL label follows the stack" $? # http_ok requests the stack's readiness path and requires the stack's own # body: one local server answers like the application (/health/ready) and # the fixture (/health); another answers every path with an SPA page. -cat >"$work/ready_server.py" <<'PYSERVER' -import http.server, sys +printf '%s\n' 'import http.server, sys mode = sys.argv[2] class H(http.server.BaseHTTPRequestHandler): def do_GET(self): if mode == "spa": status, body = 200, b"
" elif self.path == "/health/ready": - status, body = 200, b'{"status":"ready"}' + status, body = 200, b'\''{"status":"ready"}'\'' elif self.path == "/health": status, body = 200, b"ok\n" else: @@ -721,8 +710,7 @@ class H(http.server.BaseHTTPRequestHandler): pass server = http.server.HTTPServer(("127.0.0.1", 0), H) open(sys.argv[1], "w").write(str(server.server_address[1])) -server.serve_forever() -PYSERVER +server.serve_forever()' >"$work/ready_server.py" python3 "$work/ready_server.py" "$work/ready.port" stacks & bg_pids="$bg_pids $!" python3 "$work/ready_server.py" "$work/spa.port" spa & @@ -1101,8 +1089,7 @@ RUNTIME_DIR=$saved_runtime GATE_FILE=$saved_gate GATE_DIR=$saved_gate_dir # --- U-21 gate_blocks reads the edge's real answer (FND-009E) # Against one-shot local listeners: only the gate's own 503 counts as # blocked, not any other 503 and not a 200. -cat >"$work/u21-serve.py" <<'PY' -import socket, sys +printf '%s\n' 'import socket, sys status, body = sys.argv[2], sys.argv[3] s = socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(("127.0.0.1", 0)); s.listen(1) @@ -1124,8 +1111,7 @@ try: pass except OSError: pass -c.close(); s.close() -PY +c.close(); s.close()' >"$work/u21-serve.py" u21() { # name status body expected(0 blocked|1 not) local pf=$work/u21.port i=0 r rm -f "$pf" @@ -1414,22 +1400,21 @@ u25 "a missing password file is not a login" row "$work/u25.missing" no-file/fal # stack is refused. remove_untagged_images may remove only the untagged # images labelled with the run's project, by ID and never forced. mkdir -p "$work/fakeimg" -cat >"$work/fakeimg/docker" <<'FAKE' -#!/bin/sh +printf '%s\n' '#!/bin/sh d=$FAKE_IMG echo "$*" >>"$d/calls" -named() { awk -v i="$1" '$2 == i { f = 1 } END { exit !f }' "$d/names"; } -used() { awk -v i="$1" '$2 == i { f = 1 } END { exit !f }' "$d/containers"; } -drop_image() { awk -v i="$1" '$1 != i' "$d/images" >"$d/images.new" && mv "$d/images.new" "$d/images"; } +named() { awk -v i="$1" '\''$2 == i { f = 1 } END { exit !f }'\'' "$d/names"; } +used() { awk -v i="$1" '\''$2 == i { f = 1 } END { exit !f }'\'' "$d/containers"; } +drop_image() { awk -v i="$1" '\''$1 != i'\'' "$d/images" >"$d/images.new" && mv "$d/images.new" "$d/images"; } case "$1 $2" in - "ps -aq"*) cut -d' ' -f1 "$d/containers" ;; + "ps -aq"*) cut -d'\'' '\'' -f1 "$d/containers" ;; "inspect -f") shift 3 - for c in "$@"; do awk -v c="$c" '$1 == c { print $2 }' "$d/containers"; done ;; + for c in "$@"; do awk -v c="$c" '\''$1 == c { print $2 }'\'' "$d/containers"; done ;; "image ls") shift 2 case $1 in - --format) cut -d' ' -f1 "$d/names"; exit 0 ;; + --format) cut -d'\'' '\'' -f1 "$d/names"; exit 0 ;; esac # image IDs, filtered like Docker: dangling=true (no name) and # label=com.docker.compose.project=P. IDs in "stale" are listed as @@ -1450,36 +1435,36 @@ case "$1 $2" in done <"$d/images" ;; "image inspect") ref=$5 - id=$(awk -v r="$ref" '$1 == r { print $2 }' "$d/names") - [ -n "$id" ] || id=$(awk -v r="$ref" '$1 == r { print $1 }' "$d/images") + id=$(awk -v r="$ref" '\''$1 == r { print $2 }'\'' "$d/names") + [ -n "$id" ] || id=$(awk -v r="$ref" '\''$1 == r { print $1 }'\'' "$d/images") [ -n "$id" ] || exit 1 case $4 in - *RepoTags*) awk -v i="$id" '$2 == i { print $1 }' "$d/names" ;; + *RepoTags*) awk -v i="$id" '\''$2 == i { print $1 }'\'' "$d/names" ;; *) echo "$id" ;; esac ;; "image rm") shift 2 force=0 - case " $* " in *' -f '* | *' --force '*) force=1 ;; esac + case " $* " in *'\'' -f '\''* | *'\'' --force '\''*) force=1 ;; esac for a in "$@"; do case $a in -f | --force) continue ;; esac - id=$(awk -v r="$a" '$1 == r { print $2 }' "$d/names") + id=$(awk -v r="$a" '\''$1 == r { print $2 }'\'' "$d/names") if [ -n "$id" ] && [ "$force" = 0 ]; then # By name: only this name goes, and the image with its last name; # refused for the last name of an image a container uses. - if [ "$(awk -v i="$id" '$2 == i' "$d/names" | wc -l)" -eq 1 ] && used "$id"; then + if [ "$(awk -v i="$id" '\''$2 == i'\'' "$d/names" | wc -l)" -eq 1 ] && used "$id"; then echo "conflict: image is being used by a container" >&2 exit 1 fi - awk -v r="$a" '$1 != r' "$d/names" >"$d/names.new" && mv "$d/names.new" "$d/names" + awk -v r="$a" '\''$1 != r'\'' "$d/names" >"$d/names.new" && mv "$d/names.new" "$d/names" named "$id" || drop_image "$id" continue fi # By ID or forced: every name of the image goes, and the image. [ -n "$id" ] || id=$a - if ! named "$id" && ! awk -v i="$id" '$1 == i { f = 1 } END { exit !f }' "$d/images"; then + if ! named "$id" && ! awk -v i="$id" '\''$1 == i { f = 1 } END { exit !f }'\'' "$d/images"; then echo "No such image: $a" >&2 exit 1 fi @@ -1487,12 +1472,11 @@ case "$1 $2" in echo "conflict: unable to delete $id (must be forced)" >&2 exit 1 fi - awk -v i="$id" '$2 != i' "$d/names" >"$d/names.new" && mv "$d/names.new" "$d/names" + awk -v i="$id" '\''$2 != i'\'' "$d/names" >"$d/names.new" && mv "$d/names.new" "$d/names" drop_image "$id" done ;; *) echo "unexpected: docker $*" >&2; exit 99 ;; -esac -FAKE +esac' >"$work/fakeimg/docker" chmod +x "$work/fakeimg/docker" u26_dev=$( . "$here/../app/lib.sh" && project_of "$APP_REPO") u26_run=flowcrm-aaaaaa-test-bbbbbb @@ -1664,8 +1648,7 @@ u26_guard "no prune command of any kind" "$u26_pat_prune" # removal, one after && does not. (The command name is in a variable so that # this file has no match.) u26_dk=docker -cat >"$work/u26-fixture.sh" <&1 -f $u26_dk image rm y &>/dev/null --force -$u26_dk image rm y && rm -f z -EOF +$u26_dk image rm y && rm -f z" >"$work/u26-fixture.sh" u26_hits() { # pattern: first line numbers of the fixture's matching lines u26_lines "$work/u26-fixture.sh" | grep -E -- "$1" | cut -d: -f2 | paste -sd ' ' - } From e512ea84ff1e8eb233bb9b0318386da3bbc0016e Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 17:08:43 +0100 Subject: [PATCH 20/77] fix(test): end every process of the macOS isolation and deny shared temporary files and debugging The macOS profile no longer allows writes in /tmp: the sh-thd allowance let code in one sandbox rewrite the here-documents of bash 3.2 in another sandbox that cannot use /var/tmp. It also denies ptrace (system-debug), task ports of processes outside the sandbox (as used by debuggers, vmmap and sample) and reading other terminals' devices. no-docker.sh now hands its process to a supervisor in no-docker-probe.py. The sandbox carries a marker of the run and one common to every run (a denied read of a file in a marker directory outside the run's directory); from outside, sandbox_check() tells which processes are in it. When the command ends, or SIGINT, SIGTERM, SIGHUP or SIGQUIT arrives, the supervisor SIGKILLs every member as soon as it finds it, rechecking its start time, membership and the markers first, until none is left, without starting a process. A run refuses to start while a process left by an earlier run runs on its own (after SIGKILL of no-docker.sh). Status 125 if the processes cannot be identified or ended. Inside, the probe also checks that no here-document file can be created in /tmp and that the sandbox carries both markers. On Linux it prints the uid and gid, refuses a reachable second procfs or nsfs mount and counts a listener with a full backlog as reachable; /proc//stat is parsed without here-strings. --- scripts/ci/no-docker-probe.py | 363 +++++++++++++++++++++++++++++++++- scripts/ci/no-docker.sh | 82 ++++++-- 2 files changed, 424 insertions(+), 21 deletions(-) diff --git a/scripts/ci/no-docker-probe.py b/scripts/ci/no-docker-probe.py index 3a159f9..d9600c7 100644 --- a/scripts/ci/no-docker-probe.py +++ b/scripts/ci/no-docker-probe.py @@ -47,7 +47,18 @@ out; --seal WORK as root in the new mount namespace: make every mount reachable by path read-only, except the private file systems - and WORK, and verify it (Linux). + and WORK, and verify it (Linux); + --markers TMP make sure the marker directory common to every run exists + in the per-user temporary directory TMP, make one for a new + run and print its path (macOS); + --orphans TMP fail if a process of an earlier run is still alive on its + own (macOS); + --supervise MARKDIR STARTED REMOVE -- COMMAND... + run COMMAND (sandbox-exec), then end every process left in + the sandbox of MARKDIR; signals end it at once; 125 unless + the file STARTED exists; removes MARKDIR and the directory + REMOVE ("-": none) (macOS, no-docker.sh's last step); + --end MARKDIR end every process of the sandbox of MARKDIR (macOS). """ import concurrent.futures @@ -57,6 +68,8 @@ import json import os import re +import shutil +import signal import socket import stat import subprocess @@ -131,7 +144,12 @@ def connects(address): def create_error(directory): """None if a new file can be created in directory (it is removed again), else the errno of the failure.""" - path = os.path.join(directory, f".no-docker-probe-{os.getpid()}") + return create_error_named(directory, f".no-docker-probe-{os.getpid()}") + + +def create_error_named(directory, name): + """As create_error, for a file of the given name.""" + path = os.path.join(directory, name) try: fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) except FileExistsError: @@ -224,6 +242,320 @@ def check_mechanism_darwin(): report(not_writable(d), f"macOS sandbox denies writes {where} ({d})") tmpdir = os.environ.get("TMPDIR", "") report(bool(tmpdir) and create_error(tmpdir) is None, f"$TMPDIR of the run is writable ({tmpdir})") + # bash 3.2 keeps here-documents in /private/tmp/sh-thd-*: none of another + # bash outside may be created, replaced or rewritten from here. + e = create_error_named("/private/tmp", f"sh-thd-no-docker-probe-{os.getpid()}") + report(e is not None, "macOS sandbox denies writes to here-document files of bash 3.2 (/private/tmp/sh-thd-*)" + + ("" if e is not None else ": written")) + # The markers by which no-docker.sh finds this sandbox's processes from + # outside: reading A is denied, reading B is not. + for d in (os.environ.get("NO_DOCKER_MARK", ""), os.path.join(user_tmp, COMMON_MARK) if user_tmp else ""): + result = "unknown" + if d: + try: + open(os.path.join(d, "b"), "rb").close() + try: + open(os.path.join(d, "a"), "rb").close() + result = "not denied" + except PermissionError: + result = "ok" + except OSError as err: + result = errno.errorcode.get(err.errno, str(err)) + report(result == "ok", f"the sandbox carries the marker of {d or '?'} ({result})") + + +# The processes of one sandbox, seen from outside it. no-docker.sh (and the +# suite's own guard) add to the profile `(deny file-read-data (literal A))` +# for the file A of the run's marker directory and of the directory common +# to every run, and leave the file B next to it alone; no other profile +# names these files. sandbox_check() asks the kernel how a process's +# sandbox would decide, so a process is in the run's sandbox exactly when +# it is denied A and allowed B (measured: an unsandboxed process is +# allowed both, other sandboxes deny or allow both; the files must exist, +# or every process is "denied"). A process cannot leave its sandbox, and +# every process it starts inherits it. The marker directories are in the +# per-user temporary directory, outside every directory the sandbox may +# write, so the command cannot remove or replace them. +COMMON_MARK = "no-docker-mark" +SZOMB = 5 + + +class ProcBsdInfo(ctypes.Structure): + _fields_ = [("flags", ctypes.c_uint32), ("status", ctypes.c_uint32), ("xstatus", ctypes.c_uint32), + ("pid", ctypes.c_uint32), ("ppid", ctypes.c_uint32), ("uid", ctypes.c_uint32), + ("gid", ctypes.c_uint32), ("ruid", ctypes.c_uint32), ("rgid", ctypes.c_uint32), + ("svuid", ctypes.c_uint32), ("svgid", ctypes.c_uint32), ("rfu_1", ctypes.c_uint32), + ("comm", ctypes.c_char * 16), ("name", ctypes.c_char * 32), ("nfiles", ctypes.c_uint32), + ("pgid", ctypes.c_uint32), ("pjobc", ctypes.c_uint32), ("e_tdev", ctypes.c_uint32), + ("e_tpgid", ctypes.c_uint32), ("nice", ctypes.c_int32), ("start_tvsec", ctypes.c_uint64), + ("start_tvusec", ctypes.c_uint64)] + + +def private_dir(path): + """True if path is a directory of ours, mode 0700, not a symbolic link.""" + st = os.lstat(path) + return stat.S_ISDIR(st.st_mode) and st.st_uid == os.getuid() and stat.S_IMODE(st.st_mode) == 0o700 + + +def marker_files(directory, create): + """The marker files A and B of directory, which must be a private + directory with two empty regular files of ours; created if asked.""" + if create: + try: + os.mkdir(directory, 0o700) + except FileExistsError: + pass + if not private_dir(directory): + raise OSError(f"{directory} is not a directory of ours with mode 0700") + files = [] + for name in ("a", "b"): + path = os.path.join(directory, name) + if create: + try: + os.close(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)) + except FileExistsError: + pass + st = os.lstat(path) + if not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid(): + raise OSError(f"{path} is not a regular file of ours") + files.append(os.fsencode(path)) + return files + + +class Sandboxes: + """sandbox_check() and the process table (libproc). Raises OSError if + either is unavailable or gives an answer it cannot interpret, so every + caller fails closed.""" + + PROC_PIDTBSDINFO = 3 + FILTER_PATH = 1 + + def __init__(self, directory): + self.directory = directory + self.deny, self.allow = marker_files(directory, create=False) + try: + sb = ctypes.CDLL("/usr/lib/system/libsystem_sandbox.dylib", use_errno=True) + self.proc = ctypes.CDLL("/usr/lib/libproc.dylib", use_errno=True) + self.check_fn = sb.sandbox_check + no_report = ctypes.c_int.in_dll(sb, "SANDBOX_CHECK_NO_REPORT").value + except (OSError, AttributeError, ValueError) as e: + raise OSError(f"sandbox_check() is not available: {e}") from None + # Variadic after the third parameter: with exactly three argtypes, + # ctypes passes the path as a variadic argument (on the stack on + # arm64), as the function expects. + self.check_fn.restype = ctypes.c_int + self.check_fn.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int] + self.kind = self.FILTER_PATH | no_report + self.proc.proc_listallpids.argtypes = [ctypes.c_void_p, ctypes.c_int] + self.proc.proc_pidinfo.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64, ctypes.c_void_p, ctypes.c_int] + self.verify() + + def verify(self): + """The marker files still exist and this (unsandboxed) process is + allowed both and is not sandboxed. A missing file reads as denied + for every process, so this runs before every scan and every signal; + anything else means the calls do not work as measured.""" + marker_files(self.directory, create=False) + me = os.getpid() + if (self._check(me, self.deny), self._check(me, self.allow), self.check_fn(me, None, 0)) != (0, 0, 0): + raise OSError("sandbox_check() does not answer as expected for this process") + + def _check(self, pid, path): + return self.check_fn(pid, b"file-read-data", self.kind, ctypes.c_char_p(path)) + + def member(self, pid): + """True or False; raises OSError if it cannot be told. A member must + be sandboxed at all, be denied A and be allowed B.""" + sandboxed = self.check_fn(pid, None, 0) + a, b = self._check(pid, self.deny), self._check(pid, self.allow) + if sandboxed not in (0, 1) or a not in (0, 1) or b not in (0, 1): + raise OSError(f"sandbox_check() failed for pid {pid} ({sandboxed}, {a}, {b})") + return sandboxed == 1 and a == 1 and b == 0 + + def pids(self): + n = self.proc.proc_listallpids(None, 0) + if n <= 0: + raise OSError("cannot list the processes") + buf = (ctypes.c_int * (n + 1024))() + n = self.proc.proc_listallpids(buf, ctypes.sizeof(buf)) + if n <= 0: + raise OSError("cannot list the processes") + return [buf[i] for i in range(n) if buf[i] > 0 and buf[i] != os.getpid()] + + def info(self, pid): + """(ppid, uid, status, start time) of pid, or None if it is gone.""" + i = ProcBsdInfo() + if self.proc.proc_pidinfo(pid, self.PROC_PIDTBSDINFO, 0, ctypes.byref(i), ctypes.sizeof(i)) != ctypes.sizeof(i): + return None + return i.ppid, i.uid, i.status, (i.start_tvsec, i.start_tvusec) + + def members(self): + """{pid: info} of the members that still run (a zombie has ended).""" + self.verify() + found = {} + for pid in self.pids(): + inf = self.info(pid) + if inf is None or inf[2] == SZOMB: + continue + try: + if self.member(pid): + found[pid] = inf + except OSError: + if self.info(pid) is not None: + raise + return found + + +def darwin_markers(tmp): + """For no-docker.sh and the suite: make sure the common marker directory + exists in the per-user temporary directory tmp, make a new one for the + run, and print the run's.""" + marker_files(os.path.join(tmp, COMMON_MARK), create=True) + run = tempfile.mkdtemp(prefix=COMMON_MARK + ".", dir=tmp) + marker_files(run, create=True) + print(run) + return 0 + + +def darwin_orphans(tmp): + """Before a sandbox starts: fail if a process of an earlier run is still + alive on its own, that is a member of the common marker whose topmost + member ancestor was handed to launchd (no live supervisor above it). A + run in progress elsewhere keeps its members under its supervisor; its + daemons count as orphans, so a concurrent run that leaves one blocks + new ones until it ends.""" + sbx = Sandboxes(os.path.join(tmp, COMMON_MARK)) + found = sbx.members() + orphans = [] + for pid, inf in found.items(): + top, seen = inf, 0 + while top[0] in found and seen < 4096: + top = found[top[0]] + seen += 1 + if top[0] <= 1: + orphans.append(pid) + if orphans: + print("no-docker: processes of an earlier isolated run are still running (pids " + + " ".join(str(p) for p in sorted(orphans)) + "); end them first", file=sys.stderr) + return 1 + return 0 + + +def darwin_end(sbx, seconds=10.0): + """End every process of the run's sandbox: SIGKILL each member as soon as + it is found (its start time and membership read again right before the + signal, so a reused pid is not hit), and list the table again until it + has none. A killed process cannot start another, and nothing a member + does can undo SIGKILL. Needs no new process. Returns the number of + processes ended; raises OSError if members are left after the time + limit (another user's, such as a sudo inside, cannot be signalled).""" + deadline = time.monotonic() + seconds + ended = 0 + while True: + found = sbx.members() + if not found: + return ended + if time.monotonic() > deadline: + raise OSError("processes of the isolation survive (pids " + + " ".join(f"{p}:uid{inf[1]}" for p, inf in sorted(found.items())) + ")") + for pid, inf in found.items(): + now = sbx.info(pid) + sbx.verify() + if now is None or now[3] != inf[3] or not sbx.member(pid): + continue + try: + os.kill(pid, signal.SIGKILL) + ended += 1 + except (ProcessLookupError, PermissionError): + pass + time.sleep(0.01) + + +class Interrupted(Exception): + pass + + +def darwin_supervise(markdir, started_file, remove, argv): + """no-docker.sh's last step on macOS, in its process: run argv (the + sandbox-exec command line), wait for it, then end whatever is left in + its sandbox; SIGINT, SIGTERM, SIGHUP or SIGQUIT end it at once the same + way. Everything that needs a library is loaded before the command + starts, and ending it needs no new process, so a command that uses up + the processes it may start cannot prevent it. Removes the run's marker + directory and the directory remove (if not "-"). The status is the + command's, or 125 if the isolation could not be applied (started_file, + which the command's start creates, is missing) or its processes could + not all be ended; after a signal, 128 plus its number.""" + def refused(why): + print(f"no-docker: refused: {why}", file=sys.stderr) + return 125 + + try: + sbx = Sandboxes(markdir) + except OSError as e: + shutil.rmtree(markdir, ignore_errors=True) + if remove != "-": + shutil.rmtree(remove, ignore_errors=True) + return refused(f"cannot identify the processes of the isolation: {e}; the command was not run") + caught = [] + waiting = [False] + + def on_signal(signum, frame): + # Only the wait for the command is cut short; ending the processes + # always runs to its end. + caught.append(signum) + if waiting[0]: + waiting[0] = False + raise Interrupted() + + # A signal the caller ignores stays ignored, for the command too. + for s in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT): + if signal.getsignal(s) != signal.SIG_IGN: + signal.signal(s, on_signal) + child = None + status = 125 + ended, failure = 0, None + try: + try: + child = subprocess.Popen(argv, close_fds=False) + except OSError as e: + print(f"no-docker: cannot start the isolation: {e}", file=sys.stderr) + try: + # A signal that came while it was being started is not lost: the + # wait is skipped. + waiting[0] = True + if child is not None and not caught: + status = child.wait() + except Interrupted: + pass + finally: + waiting[0] = False + try: + ended = darwin_end(sbx) + if child is not None: + if child.poll() is None: + # Interrupted before sandbox-exec applied the profile: + # our own child, not yet reaped, so its pid is its own. + child.kill() + status = child.wait(timeout=10) + ended += darwin_end(sbx) + except (OSError, subprocess.TimeoutExpired) as e: + failure = str(e) + finally: + started = os.path.exists(started_file) + shutil.rmtree(markdir, ignore_errors=True) + if remove != "-": + shutil.rmtree(remove, ignore_errors=True) + if ended: + print(f"no-docker: ended {ended} processes the command left running", file=sys.stderr) + if failure: + return refused(f"processes of the isolated command could not all be ended: {failure}") + if not started: + return refused(f"the isolation tool returned {status} without starting the command; the command was not run") + if caught: + print(f"no-docker: interrupted by signal {caught[0]}; every process of the command was ended", file=sys.stderr) + return 128 + caught[0] + return 128 - status if status < 0 else status # --- Linux ------------------------------------------------------------------- @@ -360,10 +692,12 @@ def unix_connects(path): s.connect(path) return name except socket.timeout: - return name + " (timeout)" # a listener with a full backlog + return name + " (timeout)" except OSError as e: if e.errno == errno.EPROTOTYPE: continue + if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK): + return name + " (backlog full)" # a listener, busy return None finally: s.close() @@ -393,6 +727,10 @@ def check_mechanism_linux(): "no capabilities (" + " ".join(f"{k}={v}" for k, v in caps.items()) + ")") report(st.get("NoNewPrivs") == "1", f"no_new_privs is set ({st.get('NoNewPrivs', 'unknown')})") report(st.get("Groups", "x") == "", f"no supplementary groups ({st.get('Groups') or 'none'})") + # The primary group is the caller's (on a runner possibly the engine + # socket's group): what keeps the socket away is that its path is + # hidden and that no listed socket accepts a connection (below). + print(f"no-docker: info uids {st.get('Uid', '?')}; gids {st.get('Gid', '?')}") host_ns = env_pairs("NO_DOCKER_HOST_NS") for ns in ("net", "mnt", "pid", "ipc"): @@ -450,6 +788,11 @@ def check_mechanism_linux(): host = host_devs.get(path, "") report(bool(m) and m["fstype"] == fstype and bool(host) and dev != host, f"{path} is a private {fstype} (device {dev}; caller {host or 'unknown'})") + # A second procfs, or a namespace file, reachable by path would lead to + # the host's processes (/proc//root) or namespaces. + extra = [f"{m['point']} ({m['fstype']})" for m in vis.values() + if (m["fstype"] == "proc" and m["point"] != "/proc") or m["fstype"] == "nsfs"] + report(not extra, "no other proc or nsfs mount is reachable" + (f" ({' '.join(extra)})" if extra else "")) m = vis.get("/sys/fs/cgroup") report(bool(m) and m["fstype"] == "tmpfs" and "ro" in m["opts"], "/sys/fs/cgroup is masked by a read-only tmpfs") if work: @@ -610,6 +953,20 @@ def main(): sys.stdout.buffer.write(b"".join(p + b"\0" for p in paths)) print(f"no-docker: {len(paths)} Unix sockets listed on {points} mounts in {took:.1f}s", file=sys.stderr) return 0 + if sys.platform == "darwin" and sys.argv[1:2] in (["--markers"], ["--orphans"], ["--end"]) and len(sys.argv) == 3: + try: + if sys.argv[1] == "--markers": + return darwin_markers(sys.argv[2]) + if sys.argv[1] == "--orphans": + return darwin_orphans(sys.argv[2]) + n = darwin_end(Sandboxes(sys.argv[2])) + print(f"no-docker: ended {n} processes", file=sys.stderr) + return 0 + except OSError as e: + print(f"no-docker: cannot check the processes of the isolation: {e}", file=sys.stderr) + return 1 + if sys.platform == "darwin" and sys.argv[1:2] == ["--supervise"] and len(sys.argv) > 6 and sys.argv[5] == "--": + return darwin_supervise(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[6:]) if sys.argv[1:2] == ["--seal"] and len(sys.argv) == 3: try: return seal(sys.argv[2]) diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh index dab110c..1a2ebd7 100755 --- a/scripts/ci/no-docker.sh +++ b/scripts/ci/no-docker.sh @@ -26,13 +26,18 @@ # macOS sandbox-exec: no connect to a Unix socket, no TCP beyond # loopback, to loopback ports 22, 2375 and 2376 or to a loopback # port that was listening before the command started; file writes -# only in the run's own directory (the command's $TMPDIR is in it), -# the /dev nodes a shell needs and bash 3.2's here-document files in -# /tmp; no signal to a process outside the sandbox; no preference -# write, TIOCSTI, open, osascript, launchctl, ssh, launchd job, Launch -# Services, Apple Events, login item or background task registration -# and no lookup of Docker's own services. $TMPDIR must be the -# per-user directory under /private/var/folders. +# only in the run's own directory (the command's $TMPDIR is in it) +# and the /dev nodes a shell needs, nothing in /tmp (so no bash 3.2 +# here-document inside); no signal to and no debugging of a +# process outside the sandbox; no preference write, TIOCSTI, open, +# osascript, launchctl, ssh, launchd job, Launch Services, Apple +# Events, login item or background task registration and no lookup +# of Docker's own services. When the command ends, or no-docker.sh +# gets SIGINT, SIGTERM, SIGHUP or SIGQUIT, every process left in the +# sandbox is killed (found by a marker of the run); a process left +# by a run that could not do that (SIGKILL) keeps new runs from +# starting. $TMPDIR must be the per-user directory under +# /private/var/folders. # Linux sudo unshare: new network (loopback only), mount, pid, and IPC # namespaces; empty tmpfs over /run, /var/run, /tmp, /var/tmp, # /dev/shm and /sys/fs/cgroup, a fresh /dev/mqueue, every other mount @@ -53,7 +58,8 @@ # inside that each mechanism is in effect and that no engine answers on any # endpoint it can find. If isolation cannot be applied or proven, COMMAND # does not run and the exit status is 125; so is the status if the -# isolation tool returns without the command having started inside it. +# isolation tool returns without the command having started inside it, and +# on macOS if the processes the command left could not all be ended. set -u @@ -118,7 +124,7 @@ owner_mode() { # script. Called from the INT/TERM/HUP trap. # shellcheck disable=SC2329 kill_isolation() { - local d p q st n + local d p q st n rest for d in /proc/[0-9]*; do p=${d#/proc/} [ "$(cat "$d/comm" 2>/dev/null)" = unshare ] || continue @@ -126,7 +132,10 @@ kill_isolation() { n=0 while [ "$n" -lt 16 ]; do st=$(cat "/proc/$q/stat" 2>/dev/null) || break - read -r _ q _ <<<"${st##*) }" + # Field 4, the parent: after the name, "STATE PPID ...". + rest=${st##*) } + rest=${rest#* } + q=${rest%% *} case $q in "" | *[!0-9]*) break ;; esac [ "$q" -gt 1 ] || break if [ "$q" = "$spid" ]; then @@ -210,7 +219,9 @@ case $os in break fi st=$(cat "/proc/$pid/stat" 2>/dev/null) || break - read -r _ pid _ <<<"${st##*) }" + rest=${st##*) } + rest=${rest#* } + pid=${rest%% *} case $pid in "" | *[!0-9]*) break ;; esac n=$((n + 1)) done @@ -295,11 +306,32 @@ case $os in done envargs+=("NO_DOCKER_HELD_PORTS=$(printf '%s' "$listening" | tr '\n' ' ')" "NO_DOCKER_OUTER_PID=$$" "NO_DOCKER_USER_TMP=$tmp") + # The sandbox carries a marker of this run and one common to every run + # (a denied read of a file in each marker directory, which are in + # $tmp, outside everything the sandbox may write), by which + # no-docker-probe.py finds its processes from outside: a process cannot + # leave its sandbox, and every process it starts inherits it. + markdir=$(python3 -I "$probe" --markers "$tmp") || refuse "cannot make the marker of the run" + case $markdir in "$tmp"/no-docker-mark.?*) ;; *) refuse "unexpected marker directory $markdir" ;; esac + plain "$markdir" || refuse "unexpected character in the marker directory $markdir" + if [ -n "$given_work" ]; then + remove=- + trap 'rm -rf "$markdir"' EXIT + else + remove=$work + trap 'rm -rf "$work" "$markdir"' EXIT + fi + python3 -I "$probe" --orphans "$tmp" || refuse "cannot rule out a process left by an earlier isolated run" + envargs+=("NO_DOCKER_MARK=$markdir") # File writes: the run's directory; of /dev only what shells and - # interpreters open for writing (measured, docs/dev-environment.md); - # bash 3.2's here-document files, which it creates in /tmp whatever - # $TMPDIR is. Signals only within this sandbox. An SBPL error makes - # sandbox-exec fail, so the command never starts (125). + # interpreters open for writing (measured, docs/dev-environment.md). + # Nothing in /tmp: bash 3.2 keeps here-documents there when it cannot + # use /var/tmp, whatever $TMPDIR is, so code run inside must not use + # here-documents or here-strings, and no bash in another sandbox can + # have its here-documents changed from inside. Signals, task ports + # (debuggers, vmmap, sample) and process names only within this + # sandbox; no ptrace of any process; no reading other terminals. An + # SBPL error makes sandbox-exec fail, so the command never starts (125). profile="(version 1) (allow default) (deny network-outbound (remote unix-socket)) @@ -307,7 +339,9 @@ case $os in (allow network-outbound (remote ip \"localhost:*\")) (deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")$held) (deny file-write*) -(allow file-write* (subpath \"$work\") (literal \"/dev/null\") (literal \"/dev/zero\") (literal \"/dev/tty\") (literal \"/dev/dtracehelper\") (regex #\"^/dev/fd/[0-9]+\$\") (literal \"/private/tmp\") (regex #\"^/private/tmp/sh-thd[^/]*\$\")) +(allow file-write* (subpath \"$work\") (literal \"/dev/null\") (literal \"/dev/zero\") (literal \"/dev/tty\") (literal \"/dev/dtracehelper\") (regex #\"^/dev/fd/[0-9]+\$\")) +(deny file-read-data (literal \"$tmp/no-docker-mark/a\") (literal \"$markdir/a\")) +(deny file-read* (regex #\"^/dev/ttys[0-9]+\$\")) (deny user-preference-write) (deny file-ioctl (ioctl-command #x80017472)) (deny process-exec (literal \"/usr/bin/open\") (literal \"/usr/bin/osascript\") (literal \"/bin/launchctl\") (literal \"/usr/bin/ssh\")) @@ -315,10 +349,22 @@ case $os in (deny lsopen) (deny appleevent-send) (deny mach-lookup (global-name \"com.apple.coreservices.launchservicesd\") (global-name \"com.apple.coreservices.appleevents\") (global-name-regex #\"^com\\.apple\\.lsd\\.\") (global-name \"com.apple.xpc.smd\") (global-name-regex #\"^com\\.apple\\.backgroundtaskmanagement\") (global-name-regex #\"^com\\.apple\\.coreservices\\.sharedfilelistd\") (global-name-regex #\"^com\\.docker\\.\") (global-name-regex #\"^com\\.electron\\.dockerdesktop\\.\")) +(deny system-debug) +(deny mach-task-name mach-task-read mach-task-inspect mach-priv-task-port) +(allow mach-task-name mach-task-read mach-task-inspect mach-priv-task-port (target same-sandbox)) (deny signal) (allow signal (target same-sandbox))" - sandbox-exec -p "$profile" /usr/bin/env -i "${envargs[@]}" "$BASH" "$here/no-docker.sh" --inner "$@" - rc=$? + # The supervisor takes over this process: a signal sent to it ends the + # command and everything it left at once, and so does the command's + # end. It removes the run's directory (unless given with --work) and + # the marker directory. + trap - EXIT + shopt -s execfail + exec python3 -I "$probe" --supervise "$markdir" "$work/started" "$remove" -- /usr/bin/sandbox-exec -p "$profile" \ + /usr/bin/env -i "${envargs[@]}" "$BASH" "$here/no-docker.sh" --inner "$@" + rm -rf "$markdir" + [ "$remove" = - ] || rm -rf "$work" + refuse "cannot start the supervisor (python3)" ;; Linux) # Every Unix socket on the local file systems and every bound one of From a1a3eafc3534817dfb6a7aee3c8e885c6e241e0f Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 17:08:43 +0100 Subject: [PATCH 21/77] test(ci): pin the isolation suite's helpers and cover the new macOS limits canary.py, drive.sh and the mutant payload are tracked files whose sha256 values are constants of the suite, checked on every copy before it runs outside the isolation, instead of hashes taken after the suite wrote them. The suite's own guard sandbox loses the /tmp allowance, gets the same markers, rules and supervisor as no-docker.sh. New cases: T-06 now expects a bash 3.2 here-document to fail inside; T-20 (no here-document in code run inside), T-21 (here-document files in /tmp and /var/tmp cannot be changed or created), T-22 (no change through links, rename or exchange), T-23 (pinned helpers), T-24 and T-25 (no process outlives a normal end, a failure, SIGKILL of the command, or SIGINT, SIGTERM and SIGHUP to no-docker.sh), T-26 (after SIGKILL of no-docker.sh new runs refuse until the leftovers end), T-27 (refusal when the probe cannot find or end the processes), T-28 (no debugger attach, task port or memory map of an outside process) and T-29 (no reading of another terminal). --- scripts/ci/tests/no-docker.test.sh | 569 ++++++++++++++++++++------- scripts/ci/tests/no-docker/canary.py | 34 ++ scripts/ci/tests/no-docker/drive.sh | 16 + scripts/ci/tests/no-docker/payload | 37 ++ 4 files changed, 521 insertions(+), 135 deletions(-) create mode 100644 scripts/ci/tests/no-docker/canary.py create mode 100644 scripts/ci/tests/no-docker/drive.sh create mode 100644 scripts/ci/tests/no-docker/payload diff --git a/scripts/ci/tests/no-docker.test.sh b/scripts/ci/tests/no-docker.test.sh index 59a2fe7..ed8dcc0 100755 --- a/scripts/ci/tests/no-docker.test.sh +++ b/scripts/ci/tests/no-docker.test.sh @@ -31,20 +31,26 @@ # tested behind the guard. Should that isolation fail, the read-only # payload would reach the real daemon, which the cases detect; # - the helpers this suite runs outside any isolation (drive.sh, -# canary.py) and the payload are kept outside every directory the code -# under test can write, and their sha256 is checked immediately before -# each such run (T-14); +# canary.py) and the payload are tracked files (no-docker/) pinned to +# reviewed sha256 values in this file; their copies are kept outside +# every directory the code under test can write, and checked against +# those values immediately before each such run (T-14, T-23); # - the other cases run plain commands that only print, connect to this # suite's own listeners, write marker files in directories of this # suite (and in protected places, where the write must fail; a marker # that was written anyway is removed), send signal 0 or TERM to this -# suite's own `sleep` canaries, or sleep. +# suite's own `sleep` canaries, start processes of their own that +# sleep and must be ended by no-docker.sh (any left over are killed by +# pid only while their arguments carry the case's marker), attach a +# debugger to or read the memory map of this suite's own canary copies +# of /bin/sleep, read a pseudo-terminal of their own, or sleep. # This suite itself runs no docker command. # -# T-11 to T-15 are macOS only, T-16 to T-18 Linux only (each prints INFO on -# the other platform); T-19 is static and runs on both. Each case says how -# it fails against the no-docker.sh of 38b5ebd, which these cases were -# written to catch. +# T-06, T-08 to T-15, T-21, T-22 and T-24 to T-29 are macOS only, T-16 to +# T-18 Linux only (each prints INFO on the other platform); T-19, T-20 and +# T-23 are static and run on both. Each case says how it fails against the +# no-docker.sh of 38b5ebd or dfb6ee4, which these cases were written to +# catch. # # Usage: no-docker.test.sh # HARNESS= checks another copy of tests/devenv/unit.sh instead of the @@ -93,6 +99,9 @@ nonce=$$-$(date +%s) home_marker=$home/.flowcrm-no-docker-test-$nonce tmp=$(cd -P "${TMPDIR:-/tmp}" && pwd -P) [ -n "$tmp" ] && [ "$tmp" != / ] || exit 2 +# macOS: the per-user temporary directory no-docker.sh uses (its markers). +ndtmp=$tmp +[ "$os" != Darwin ] || [ -n "${TMPDIR-}" ] || ndtmp=$(cd -P "$(getconf DARWIN_USER_TEMP_DIR)" && pwd -P) || exit 2 # Next to $TMPDIR: writable by the user, outside what the sandbox allows. outside_marker=$(dirname "$tmp")/.flowcrm-no-docker-test-$nonce tmp_marker=/private/tmp/.flowcrm-no-docker-test-$nonce @@ -133,66 +142,20 @@ new_work() { # a fresh, empty directory for --work mktemp -d "$vt/w.XXXXXX" } -# --- the canary engine -------------------------------------------------------- -cat >"$helpers/canary.py" <<'PY' -import json, os, socketserver, sys -path, log = sys.argv[1], sys.argv[2] -class H(socketserver.StreamRequestHandler): - def handle(self): - while True: - line = self.rfile.readline() - if not line: - return - headers = {} - while True: - h = self.rfile.readline() - if h in (b"\r\n", b"\n", b""): - break - k, _, v = h.decode("latin-1").partition(":") - headers[k.strip().lower()] = v.strip() - if headers.get("user-agent") != "no-docker-probe": - with open(log, "a") as f: - f.write(line.decode("latin-1").strip() + " ua=" + headers.get("user-agent", "-") + "\n") - target = line.split()[1].decode("latin-1") if len(line.split()) > 1 else "/" - if target.endswith("/_ping"): - body = b"OK" - elif target.endswith("/version"): - body = json.dumps({"Version": "canary-engine", "ApiVersion": "1.45", "MinAPIVersion": "1.24", - "Os": "linux", "Arch": "amd64"}).encode() - else: - body = json.dumps({"message": "canary-engine"}).encode() - self.wfile.write(b"HTTP/1.1 200 OK\r\nApi-Version: 1.45\r\nOstype: linux\r\n" - b"Content-Type: application/json\r\nContent-Length: %d\r\n\r\n" % len(body)) - if not line.startswith(b"HEAD"): - self.wfile.write(body) - self.wfile.flush() -class S(socketserver.ThreadingMixIn, socketserver.UnixStreamServer): - daemon_threads = True -S(path, H).serve_forever() -PY - -# drive.sh : -# starts the canary (logging to /canary.log), records that the -# harness was started (/drive-started), runs it with DOCKER_HOST -# pointing at the canary and stops the canary again. -cat >"$helpers/drive.sh" <<'DRIVE' -canary_py=$1 data=$2 canary=$3 -shift 3 -mkdir -p "$data" || exit 2 -: >"$data/canary.log" || exit 2 -rm -f "$canary" -python3 -I "$canary_py" "$canary" "$data/canary.log" & -cpid=$! -for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do [ -S "$canary" ] && break; sleep 0.2; done -[ -S "$canary" ] || { echo "HARNESS ERROR: the canary engine did not start"; kill "$cpid"; exit 2; } -: >"$data/drive-started" -DOCKER_HOST=unix://$canary "$@" -rc=$? -sleep 0.5 # let the canary finish writing -kill "$cpid" 2>/dev/null -wait "$cpid" 2>/dev/null -exit "$rc" -DRIVE +# --- the helpers ------------------------------------------------------------- +# Tracked files of this suite (scripts/ci/tests/no-docker/), copied to +# $helpers and checked against the sha256 below, which is part of this +# reviewed source: a copy that differs in any way is never run. Changing +# a helper needs a new review and a new hash here. +# canary.py : the canary engine, a Unix socket that answers +# like Docker and records every request; +# drive.sh : +# starts the canary (logging to /canary.log), records that +# the harness was started (/drive-started), runs it with +# DOCKER_HOST pointing at the canary and stops the canary again; +# payload: the mutant's payload (below). +helper_sums="canary.py=35dae9af68f8ec8a8a8668a1468fb192ed14f35461d893e3ab9525db11d69ce5 drive.sh=bb13bfa8771d611cc6be3bacf709b769b0c7d0868f0d2821b040107019a7d283 payload=7fd727d79f1eb2e8e3a6f52e25da8f0fe35851b79d262384f6bbc5f5c946ed25" +cp "$here/no-docker/canary.py" "$here/no-docker/drive.sh" "$here/no-docker/payload" "$helpers/" || exit 2 # A Docker configuration whose context "canary" points at a canary socket. ctx=$(printf 'canary' | { if command -v sha256sum >/dev/null 2>&1; then sha256sum; else shasum -a 256; fi; } | cut -c1-64) @@ -203,62 +166,16 @@ ctxcfg() { # dir canary } # --- the mutant --------------------------------------------------------------- -# Inserted after the first line of bin/devenv, so it runs as soon as the file -# is sourced or executed. Each case appends " rc= out=" -# to the log; the mutant then exits 97. -cat >"$helpers/payload" <<'MUT' -qa_log=@LOG@ qa_canary=@CANARY@ qa_cfg=@CFG@ qa_marker=@MARKER@ -qa() { # case command... - qa_n=$1 - shift - qa_out=$("$@" 2>&1) - printf '%s rc=%s out=%s\n' "$qa_n" "$?" "$(printf '%s' "$qa_out" | tr '\n' ' ' | cut -c1-200)" >>"$qa_log" -} -qa_fmt='{{.Server.Version}}' -qa inherited-env docker version --format "$qa_fmt" -qa path-lookup docker -H "unix://$qa_canary" version --format "$qa_fmt" -for qa_p in /usr/local/bin/docker /usr/bin/docker /opt/homebrew/bin/docker /Applications/Docker.app/Contents/Resources/bin/docker; do - [ ! -x "$qa_p" ] || qa "absolute-path:$qa_p" "$qa_p" -H "unix://$qa_canary" version --format "$qa_fmt" -done -qa builtin-command command docker -H "unix://$qa_canary" version --format "$qa_fmt" -qa env-wrapper env docker -H "unix://$qa_canary" version --format "$qa_fmt" -qa env-docker-host env DOCKER_HOST="unix://$qa_canary" docker version --format "$qa_fmt" -qa context env DOCKER_CONFIG="$qa_cfg" docker --context canary version --format "$qa_fmt" -qa nested-bash bash -c 'docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" -qa nested-sh sh -c 'docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" -qa_x=$(docker -H "unix://$qa_canary" version --format "$qa_fmt" 2>&1) -printf 'command-substitution rc=%s out=%s\n' "$?" "$(printf '%s' "$qa_x" | tr '\n' ' ')" >>"$qa_log" -qa_x=`docker -H "unix://$qa_canary" version --format "$qa_fmt" 2>&1` -printf 'backticks rc=%s out=%s\n' "$?" "$(printf '%s' "$qa_x" | tr '\n' ' ')" >>"$qa_log" -qa_dk=docker -qa variable "$qa_dk" -H "unix://$qa_canary" version --format "$qa_fmt" -qa quoted "docker" -H "unix://$qa_canary" version --format "$qa_fmt" -qa exec-subshell sh -c 'exec docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" -qa xargs sh -c 'echo version | xargs docker -H "unix://$1"' _ "$qa_canary" -(docker -H "unix://$qa_canary" version --format "$qa_fmt" >"$qa_log.bg" 2>&1; echo "background rc=$? out=$(tr '\n' ' ' <"$qa_log.bg")" >>"$qa_log") & -wait -qa sudo sudo -n docker -H "unix://$qa_canary" version --format "$qa_fmt" -qa curl-unix-socket curl -sS --max-time 5 --unix-socket "$qa_canary" http://docker/version -qa python-raw-socket python3 -c 'import socket,sys; s=socket.socket(socket.AF_UNIX); s.connect(sys.argv[1]); s.sendall(b"GET /version HTTP/1.0\r\n\r\n"); print(s.recv(4096).decode())' "$qa_canary" -qa real-socket-cli docker -H unix:///var/run/docker.sock version --format "$qa_fmt" -qa real-socket-curl curl -sS --max-time 5 --unix-socket /var/run/docker.sock http://docker/_ping -qa home-write touch "$qa_marker" -exit 97 -MUT - -# The payload is exactly the reviewed text: on macOS the cases T-03 and T-04 -# run without the outer guard, so their safety rests on it only reading. -# Changing it needs a new safety review and a new hash here. -payload_sha256=7fd727d79f1eb2e8e3a6f52e25da8f0fe35851b79d262384f6bbc5f5c946ed25 -helper_sums="canary.py=$(sha "$helpers/canary.py") drive.sh=$(sha "$helpers/drive.sh") payload=$(sha "$helpers/payload")" -case " $helper_sums " in - *" payload=$payload_sha256 "*) ;; - *) echo "HARNESS ERROR: the mutant payload differs from the reviewed one; refusing to run" >&2; exit 2 ;; -esac +# The payload is inserted after the first line of bin/devenv, so it runs as +# soon as the file is sourced or executed. Each case appends +# " rc= out=" to the log; the mutant then exits 97. +# It is exactly the reviewed text: on macOS the cases T-03 and T-04 run +# without the outer guard, so their safety rests on it only reading. # verify_helpers : the helpers and the payload in are regular -# files with the sha256 recorded when this suite wrote them. Called -# immediately before every run of drive.sh and before every mutant is made. +# files with the reviewed sha256 of helper_sums. Called right after they +# are copied, immediately before every run of drive.sh and before every +# mutant is made. verify_helpers() { local f want got for f in canary.py drive.sh payload; do @@ -270,11 +187,12 @@ verify_helpers() { fi got=$(sha "$1/$f") if [ "$got" != "$want" ]; then - echo "HARNESS ERROR: $1/$f changed since it was written ($got); refusing to run" >&2 + echo "HARNESS ERROR: $1/$f is not the reviewed one ($got); refusing to run" >&2 return 1 fi done } +verify_helpers "$helpers" || exit 2 # bare : drive.sh outside any # isolation, only after its helpers verified. @@ -344,17 +262,22 @@ run_case() { case $os-$mode in Darwin-bare) bare "$helpers" "$data" "$canary" "${cmd[@]}" ;; Darwin-guard) - # The write policy of no-docker.sh: only the case's directories, the - # /dev nodes of its profile and bash 3.2's here-documents; plus its - # other denies and the ports listening now. Only the canary socket - # is reachable. + # The write policy of no-docker.sh: only the case's directories and + # the /dev nodes of its profile; plus its other denies and the ports + # listening now. Only the canary socket is reachable. The guard's + # sandbox carries markers like no-docker.sh's, and no-docker-probe.py + # supervises it the same way: whatever the harness leaves running is + # ended when drive.sh returns, and the case fails if that cannot be + # shown (125). verify_helpers "$helpers" || exit 2 held="" for port in $(python3 -I "$root/scripts/ci/no-docker-probe.py" --listening) ; do case $port in *[!0-9]* | "") echo "HARNESS ERROR: unexpected port $port" >&2; exit 2 ;; esac held="$held (remote ip \"localhost:$port\")" done - sandbox-exec -p "(version 1) + gmark=$(python3 -I "$root/scripts/ci/no-docker-probe.py" --markers "$ndtmp") || exit 2 + case $gmark in "$ndtmp"/no-docker-mark.?*) ;; *) echo "HARNESS ERROR: unexpected marker directory $gmark" >&2; exit 2 ;; esac + python3 -I "$root/scripts/ci/no-docker-probe.py" --supervise "$gmark" "$data/drive-started" - -- /usr/bin/sandbox-exec -p "(version 1) (allow default) (deny network-outbound (remote unix-socket)) (allow network-outbound (remote unix-socket (path-literal \"$canary\"))) @@ -362,9 +285,14 @@ run_case() { (allow network-outbound (remote ip \"localhost:*\")) (deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")$held) (deny file-write*) -(allow file-write* (subpath \"$data\") (subpath \"$w\") (literal \"/dev/null\") (literal \"/dev/zero\") (literal \"/dev/tty\") (literal \"/dev/dtracehelper\") (regex #\"^/dev/fd/[0-9]+\$\") (literal \"/private/tmp\") (regex #\"^/private/tmp/sh-thd[^/]*\$\")) +(allow file-write* (subpath \"$data\") (subpath \"$w\") (literal \"/dev/null\") (literal \"/dev/zero\") (literal \"/dev/tty\") (literal \"/dev/dtracehelper\") (regex #\"^/dev/fd/[0-9]+\$\")) +(deny file-read-data (literal \"$ndtmp/no-docker-mark/a\") (literal \"$gmark/a\")) +(deny file-read* (regex #\"^/dev/ttys[0-9]+\$\")) (deny user-preference-write) (deny file-ioctl (ioctl-command #x80017472)) +(deny system-debug) +(deny mach-task-name mach-task-read mach-task-inspect mach-priv-task-port) +(allow mach-task-name mach-task-read mach-task-inspect mach-priv-task-port (target same-sandbox)) (deny signal) (allow signal (target same-sandbox)) (deny process-exec (literal \"/usr/bin/open\") (literal \"/usr/bin/osascript\") (literal \"/bin/launchctl\") (literal \"/usr/bin/ssh\")) @@ -437,15 +365,20 @@ r=$? result "T-02 claimed but unproven isolation (NO_DOCKER_ACTIVE outside it): refused, command not run" $? "(rc $r)" if [ "$os" = Darwin ]; then - # --- T-06 plain commands (no mutant): the write limits + # --- T-06 plain commands (no mutant): the write limits. A bash 3.2 + # here-document from a working directory outside the run's directory + # fails: bash keeps it in /tmp (it cannot use /var/tmp inside), which the + # sandbox does not allow, so code run inside uses none. + # On dfb6ee4: /private/tmp/sh-thd* was writable, so the here-document + # worked (rc 0 and "here-document" in t06). w6=$(new_work) || exit 2 - "$nd" --work "$w6" /bin/bash -c 'touch "$1" "$2" "$3" 2>/dev/null; cat >"$4" </dev/null; cd / && cat >"$4" </dev/null 2>&1 +EOF' _ "$home_marker" "$outside_marker" "$tmp_marker" "$w6/t06" >"$work/t06.out" 2>&1 r=$? - [ "$r" -eq 0 ] && [ "$(cat "$w6/t06" 2>/dev/null)" = here-document ] && + [ "$r" -eq 1 ] && [ ! -s "$w6/t06" ] && grep -q 'cannot create temp file for here.document' "$work/t06.out" && [ ! -e "$home_marker" ] && [ ! -e "$outside_marker" ] && [ ! -e "$tmp_marker" ] - result "T-06 writes only in the run's directory (bash 3.2 here-documents work): not under \$HOME, next to \$TMPDIR or in /tmp" $? "(rc $r)" + result "T-06 writes only in the run's directory: not under \$HOME, next to \$TMPDIR or in /tmp (a bash 3.2 here-document fails)" $? "(rc $r)" rm -f "$home_marker" "$outside_marker" "$tmp_marker" # T-08: the held-port list must be complete. A netstat without its column @@ -626,8 +559,352 @@ print("own-group", g.wait()) t15 "/tmp" /tmp t15 "/private/var/tmp itself" /private/var/tmp t15 "a relative path" t15 + + # --- T-21 here-document files of bash 3.2 cannot be changed from inside. + # Outside the sandbox bash 3.2 keeps them in /private/var/tmp, inside it + # (and in other sandboxes that allow /tmp but not /var/tmp) in + # /private/tmp. Two canaries named like them, made here: none can be + # opened for writing, truncated, appended to, removed, renamed or + # replaced from inside, and no new one can be created. + # On dfb6ee4: the profile allowed writes to /private/tmp/sh-thd*, so every + # operation on that canary succeeded. + t21a=/private/tmp/sh-thd-flowcrm-test-$nonce + t21b=/private/var/tmp/sh-thd-flowcrm-test-$nonce + w21=$(new_work) || exit 2 + for f in "$t21a" "$t21b"; do (umask 077 && printf 'original\n' >"$f") || exit 2; done + t21=$("$nd" --work "$w21" python3 -I -c ' +import errno, os, sys +w = sys.argv[1] +for path in sys.argv[2:]: + open(os.path.join(w, "x"), "w").close() + tries = { + "write": lambda: os.open(path, os.O_WRONLY), + "read-write": lambda: os.open(path, os.O_RDWR), + "append": lambda: os.open(path, os.O_WRONLY | os.O_APPEND), + "truncate": lambda: os.truncate(path, 0), + "unlink": lambda: os.unlink(path), + "rename-away": lambda: os.rename(path, path + ".moved"), + "replace": lambda: os.rename(os.path.join(w, "x"), path), + "create": lambda: os.open(path + ".new", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), + "chmod": lambda: os.chmod(path, 0o666), + } + for name, f in tries.items(): + try: + r = f() + if isinstance(r, int): + os.close(r) + print("t21 bad ", path, name, "allowed") + except OSError as e: + print("t21 " + ("ok " if e.errno in (errno.EPERM, errno.EACCES) else "bad ") , path, name, + errno.errorcode.get(e.errno, e.errno)) +' "$w21" "$t21a" "$t21b" 2>/dev/null) + r=$? + t21_left="" + for f in "$t21a" "$t21b"; do + [ "$(cat "$f" 2>/dev/null)" = original ] || t21_left="$t21_left changed:$f" + for x in "$f.moved" "$f.new"; do [ ! -e "$x" ] || t21_left="$t21_left $x"; done + done + printf '%s\n' "$t21" | grep '^t21 bad' | sed 's/^/INFO /' + [ "$r" -eq 0 ] && [ "$(printf '%s\n' "$t21" | grep -c '^t21 ok')" -eq 18 ] && [ -z "$t21_left" ] + result "T-21 bash 3.2 here-document files (/private/tmp and /private/var/tmp sh-thd-*) cannot be written, replaced or created from inside" $? \ + "(rc $r, $(printf '%s\n' "$t21" | grep -c '^t21 ok') of 18 denied, changed or left:${t21_left:- none})" + rm -f "$t21a" "$t21b" "$t21a.moved" "$t21b.moved" "$t21a.new" "$t21b.new" + + # --- T-22 a file outside the run's directory cannot be changed through a + # hard link, a symbolic link, a rename over it or an exchange (all on + # the same volume as the run's directory). + # (A-M3: measured on 38b5ebd, no regression case until now.) + t22f=$work/t22-target + printf 'original\n' >"$t22f" || exit 2 + w22=$(new_work) || exit 2 + t22=$("$nd" --work "$w22" python3 -I -c ' +import ctypes, errno, os, sys +w, target = sys.argv[1], sys.argv[2] +libc = ctypes.CDLL(None, use_errno=True) +def say(ok, text): + print(("t22 ok " if ok else "t22 bad ") + text) +try: + os.link(target, os.path.join(w, "hard")) + try: + with open(os.path.join(w, "hard"), "a") as f: + f.write("changed\n") + say(False, "hard link made and written through") + except OSError as e: + say(True, f"hard link made, but not writable ({errno.errorcode.get(e.errno)})") +except OSError as e: + say(True, f"hard link refused ({errno.errorcode.get(e.errno)})") +os.symlink(target, os.path.join(w, "soft")) +try: + with open(os.path.join(w, "soft"), "a") as f: + f.write("changed\n") + say(False, "written through a symbolic link") +except OSError as e: + say(True, f"symbolic link not writable ({errno.errorcode.get(e.errno)})") +mine = os.path.join(w, "mine") +with open(mine, "w") as f: + f.write("replaced\n") +try: + os.rename(mine, target) + say(False, "renamed over the target") +except OSError as e: + say(True, f"rename over the target refused ({errno.errorcode.get(e.errno)})") +RENAME_SWAP = 0x2 +r = libc.renamex_np(os.fsencode(mine), os.fsencode(target), RENAME_SWAP) +say(r != 0, f"exchange with the target refused ({errno.errorcode.get(ctypes.get_errno(), r)})") +r = libc.exchangedata(os.fsencode(mine), os.fsencode(target), 0) +say(r != 0, f"exchangedata with the target refused ({errno.errorcode.get(ctypes.get_errno(), r)})") +' "$w22" "$t22f" 2>/dev/null) + r=$? + printf '%s\n' "$t22" | grep '^t22 ' | sed 's/^/INFO /' + [ "$r" -eq 0 ] && [ "$(printf '%s\n' "$t22" | grep -c '^t22 ok')" -eq 5 ] && [ "$(cat "$t22f")" = original ] + result "T-22 a file outside the run's directory cannot be changed through links, a rename or an exchange" $? \ + "(rc $r, target $(cat "$t22f" 2>/dev/null | head -n 1))" + + # --- T-24 to T-27 no process of the command outlives it. The command is a + # Python "leaver" that starts four processes meant to survive it: a plain + # child, a setsid daemon after a double fork, a posix_spawn child with + # POSIX_SPAWN_SETSID and a child in a process group of its own. Each + # appends its pid to /pids once it runs and then sleeps 60 s; + # every one carries the case's marker in its arguments, which is how the + # checks below (and their clean-up) recognise them. The leaver waits until + # all four are recorded, then ends as the case says. The pids are read + # back as data and a pid counts as a survivor only while its arguments + # still carry the marker. + # On dfb6ee4: nothing ended them, so all four survived each case. + t24_leaver=' +import ctypes, os, signal, sys, time +w, tag, how = sys.argv[1], sys.argv[2], sys.argv[3] +SLEEP = [sys.executable, "-I", "-B", "-c", "import os, sys, time; open(sys.argv[1], \"a\").write(f\"{os.getpid()}\\n\"); time.sleep(60)", w + "/pids", tag] +if os.fork() == 0: + os.execv(SLEEP[0], SLEEP) +if os.fork() == 0: + os.setsid() + if os.fork() == 0: + os.execv(SLEEP[0], SLEEP) + os._exit(0) +libc = ctypes.CDLL(None, use_errno=True) +attr = ctypes.c_void_p() +libc.posix_spawnattr_init(ctypes.byref(attr)) +libc.posix_spawnattr_setflags(ctypes.byref(attr), ctypes.c_short(0x0400)) # POSIX_SPAWN_SETSID +argv = (ctypes.c_char_p * (len(SLEEP) + 1))(*[a.encode() for a in SLEEP], None) +pid = ctypes.c_int() +libc.posix_spawn(ctypes.byref(pid), SLEEP[0].encode(), None, ctypes.byref(attr), argv, None) +if os.fork() == 0: + os.setpgid(0, 0) + os.execv(SLEEP[0], SLEEP) +for _ in range(100): + try: + if len(open(w + "/pids").read().split()) >= 4: + break + except OSError: + pass + time.sleep(0.05) +open(w + "/mark", "w").write(os.environ.get("NO_DOCKER_MARK", "")) +if how == "fail": + sys.exit(3) +if how == "kill-self": + os.kill(os.getpid(), signal.SIGKILL) +if how == "wait": + open(w + "/pids", "a").write(f"{os.getpid()}\n") + time.sleep(60) +' + t24_left() { # run-dir tag: the recorded pids that still run as the case's processes + local p + regular "$1/pids" || return 0 + for p in $(tr -c '0-9\n' ' ' <"$1/pids"); do + ps -o args= -p "$p" 2>/dev/null | grep -q -- "$2" && printf ' %s' "$p" + done + } + t24_clean() { # run-dir tag: end only processes that still carry the case's marker + local p + for p in $(t24_left "$1" "$2"); do kill -KILL "$p" 2>/dev/null; done + } + t24() { # name how expected-rc + local w tag=t24-$2-$nonce left n + w=$(new_work) || exit 2 + "$nd" --work "$w" python3 -I -B -c "$t24_leaver" "$w" "$tag" "$2" >"$work/t24-$2.out" 2>&1 + r=$? + sleep 0.5 + left=$(t24_left "$w" "$tag") + n=$(count_in . "$w/pids") + t24_clean "$w" "$tag" + [ "$r" -eq "$3" ] && [ "$n" -eq 4 ] && [ -z "$left" ] && grep -q 'no-docker: ended [0-9]* processes' "$work/t24-$2.out" + result "T-24 $1: none of the 4 processes it left survives" $? "(rc $r, $n recorded, survivors:${left:- none})" + } + t24 "normal end" ok 0 + t24 "failed command" fail 3 + t24 "command killed by SIGKILL" kill-self 137 + + # T-25: SIGINT, SIGTERM or SIGHUP to no-docker.sh while the command runs + # ends the command and all it started at once. no-docker.sh is started + # with SIGINT at its default (an asynchronous command of this script + # would otherwise ignore it). + t25() { # signal expected-rc + local w tag=t25-$1-$nonce ndpid left n t0 took + w=$(new_work) || exit 2 + python3 -I -c 'import os, signal, sys +signal.signal(signal.SIGINT, signal.SIG_DFL) +os.execv(sys.argv[1], sys.argv[1:])' "$nd" --work "$w" python3 -I -B -c "$t24_leaver" "$w" "$tag" wait >"$work/t25-$1.out" 2>&1 & + ndpid=$! + for _ in $(seq 1 100); do [ "$(count_in . "$w/pids")" -ge 5 ] && break; sleep 0.1; done + n=$(count_in . "$w/pids") + t0=$(date +%s) + # Our own child, still running (its arguments name no-docker.sh). + ps -o args= -p "$ndpid" 2>/dev/null | grep -q -- no-docker && kill -"$1" "$ndpid" + wait "$ndpid" + r=$? + took=$(($(date +%s) - t0)) + sleep 0.5 + left=$(t24_left "$w" "$tag") + t24_clean "$w" "$tag" + [ "$n" -eq 5 ] && [ "$r" -eq "$2" ] && [ "$took" -le 10 ] && [ -z "$left" ] + result "T-25 SIG$1 to no-docker.sh ends the command and the 4 processes it left" $? \ + "(rc $r, $n recorded, ${took}s, survivors:${left:- none})" + } + t25 INT 130 + t25 TERM 143 + t25 HUP 129 + + # T-26: SIGKILL of no-docker.sh cannot be handled: what the command left + # keeps running, still sandboxed, and no new isolated run starts while it + # does (125, command not run). The case then ends them through the run's + # marker (no-docker-probe.py --end) and a new run starts again. + w=$(new_work) || exit 2 + tag=t26-$nonce + t26_before=$(cd "$ndtmp" && ls -d no-docker-mark.* 2>/dev/null) + python3 -I -c 'import os, signal, sys +signal.signal(signal.SIGINT, signal.SIG_DFL) +os.execv(sys.argv[1], sys.argv[1:])' "$nd" --work "$w" python3 -I -B -c "$t24_leaver" "$w" "$tag" wait >"$work/t26.out" 2>&1 & + ndpid=$! + for _ in $(seq 1 100); do [ "$(count_in . "$w/pids")" -ge 5 ] && break; sleep 0.1; done + ps -o args= -p "$ndpid" 2>/dev/null | grep -q -- no-docker && kill -KILL "$ndpid" + wait "$ndpid" 2>/dev/null + sleep 0.5 + t26_alive=$(t24_left "$w" "$tag") + w26=$(new_work) || exit 2 + "$nd" --work "$w26" /usr/bin/touch "$w26/ran" >"$work/t26b.out" 2>&1 + r=$? + t26_mark="" + regular "$w/mark" && t26_mark=$(cat "$w/mark") + # Data the sandboxed command wrote: accepted only as the name of a marker + # directory that did not exist before this case (one path component). + case ${t26_mark#"$ndtmp"/} in no-docker-mark.*[!A-Za-z0-9_]* | "$t26_mark") t26_mark="" ;; no-docker-mark.?*) ;; *) t26_mark="" ;; esac + case " +$t26_before +" in *" +${t26_mark#"$ndtmp"/} +"*) t26_mark="" ;; esac + [ -n "$t26_mark" ] && [ -d "$t26_mark" ] && [ ! -L "$t26_mark" ] || t26_mark="" + t26_end=1 + if [ -n "$t26_mark" ] && python3 -I "$root/scripts/ci/no-docker-probe.py" --end "$t26_mark" >/dev/null 2>&1; then + t26_end=0 + rm -rf "$t26_mark" + fi + t26_left=$(t24_left "$w" "$tag") + t24_clean "$w" "$tag" + w26c=$(new_work) || exit 2 + "$nd" --work "$w26c" /usr/bin/touch "$w26c/ran" >/dev/null 2>&1 + r2=$? + [ -n "$t26_alive" ] && [ "$r" -eq 125 ] && [ ! -e "$w26/ran" ] && grep -q 'earlier isolated run' "$work/t26b.out" && + [ "$t26_end" -eq 0 ] && [ -z "$t26_left" ] && [ "$r2" -eq 0 ] && [ -e "$w26c/ran" ] + result "T-26 after SIGKILL of no-docker.sh its leftovers keep every new run from starting until they are ended" $? \ + "(left after SIGKILL:${t26_alive:- none}; new run rc $r; ended through the marker $t26_end; afterwards rc $r2)" + + # T-27: if the processes of the isolation cannot be found or ended (here: + # no-docker-probe.py fails), nothing runs. + mkdir -p "$work/t27" + t27() { # mode + local w + printf '#!/bin/sh\ncase " $* " in *" %s "*) exit 1 ;; esac\nexec %s "$@"\n' "$1" "$(command -v python3)" >"$work/t27/python3" + chmod +x "$work/t27/python3" + w=$(new_work) || exit 2 + PATH=$work/t27:$PATH "$nd" --work "$w" /usr/bin/touch "$w/ran" >/dev/null 2>&1 + r=$? + [ "$r" -ne 0 ] && [ ! -e "$w/ran" ] + result "T-27 no-docker-probe.py $1 fails: the command is not run" $? "(rc $r)" + } + t27 --markers + t27 --orphans + t27 --supervise + + # --- T-28 no debugger access to a process outside (A-M2). The canaries + # are copies of /bin/sleep in this suite's directory, signed ad hoc with + # get-task-allow, so that a debugger of the same user may attach to them; + # the control attaches to one from outside and reads the other's memory + # map. Inside, ptrace(PT_ATTACHEXC) on the other must fail (EPERM), its + # task name port and its memory map (vmmap) must be refused, and it must + # still run. + # On dfb6ee4: the profile denied none of these, so the attach from inside + # succeeded (and ended the canary when the attacher exited). + t28c=$work/t28-sleep + cp /bin/sleep "$t28c" && + printf '%s\n' '' \ + '' \ + 'com.apple.security.get-task-allow' >"$work/t28.plist" && + codesign -f -s - --entitlements "$work/t28.plist" "$t28c" >/dev/null 2>&1 || exit 2 + t28_attach=' +import ctypes, errno, subprocess, sys +libc = ctypes.CDLL(None, use_errno=True) +pid = int(sys.argv[1]) +if sys.argv[2] == "inside": + me = ctypes.c_uint.in_dll(libc, "mach_task_self_").value + port = ctypes.c_uint(0) + kr = libc.task_name_for_pid(me, pid, ctypes.byref(port)) + vm = subprocess.run(["/usr/bin/vmmap", "--summary", str(pid)], capture_output=True, timeout=60).returncode + print("task-name " + ("got" if kr == 0 else "refused") + " vmmap " + ("read" if vm == 0 else "refused"), end=" ") +r = libc.ptrace(14, pid, None, 0) # PT_ATTACHEXC +print("attached" if r == 0 else "denied " + errno.errorcode.get(ctypes.get_errno(), "?")) +' + "$t28c" 60 & + t28a=$! + "$t28c" 60 & + t28b=$! + sleep 0.3 + t28_vm=$(/usr/bin/vmmap --summary "$t28b" >/dev/null 2>&1 && echo read || echo refused) + t28_out=$(python3 -I -c "$t28_attach" "$t28a" outside 2>&1) + t28_in=$("$nd" python3 -I -c "$t28_attach" "$t28b" inside 2>/dev/null) + r=$? + t28_state=$(ps -o stat= -p "$t28b" 2>/dev/null | tr -d ' ') + for p in "$t28a" "$t28b"; do + ps -o args= -p "$p" 2>/dev/null | grep -q -- "$t28c" && kill -KILL "$p" 2>/dev/null + done + wait "$t28a" "$t28b" 2>/dev/null + [ "$t28_out" = attached ] && [ "$t28_vm" = read ] && [ "$r" -eq 0 ] && + [ "$t28_in" = "task-name refused vmmap refused denied EPERM" ] && case $t28_state in S*) true ;; *) false ;; esac + result "T-28 a debugger attach, task port or memory map of a process outside is refused from inside (control: they work from outside)" $? \ + "(outside: $t28_out, vmmap $t28_vm; inside rc $r: $t28_in; canary state ${t28_state:-gone})" + + # --- T-29 another terminal cannot be read from inside: a pseudo-terminal + # this case opens outside stands for one; its device is opened by path + # (control: outside it can be, and a line written to it is read). + # On dfb6ee4: the profile allowed reading /dev/ttys*, so the line typed + # into another terminal was read. + t29=$(python3 -I -c ' +import os, pty, select, subprocess, sys +probe = "import os, select, sys\ntry:\n fd = os.open(sys.argv[1], os.O_RDONLY | os.O_NOCTTY)\nexcept OSError as e:\n print(\"denied\", e.errno)\n sys.exit(0)\nr, _, _ = select.select([fd], [], [], 5)\nprint(\"read\" if r and os.read(fd, 64) else \"opened\")\n" +out = [] +for where in ("outside", "inside"): + master, slave = pty.openpty() + argv = [sys.executable, "-I", "-c", probe, os.ttyname(slave)] + if where == "inside": + argv = [sys.argv[1]] + argv + p = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + select.select([], [], [], 1.5) + os.write(master, b"typed\n") + try: + o, _ = p.communicate(timeout=60) + except subprocess.TimeoutExpired: + p.kill() + o, _ = p.communicate() + out.append(f"{where}:{o.strip()}") + os.close(master) + os.close(slave) +print(" ".join(out)) +' "$nd" 2>&1) + [ "$t29" = "outside:read inside:denied 1" ] + result "T-29 another terminal's device cannot be read from inside (control: it can from outside)" $? "($t29)" else - echo "INFO T-06, T-08 to T-15 skipped: macOS sandbox profile only" + echo "INFO T-06, T-08 to T-15, T-21, T-22, T-24 to T-29 skipped: macOS sandbox profile only" fi # --- T-07 loopback ports listening before the isolation @@ -882,6 +1159,28 @@ else echo "INFO T-16, T-17, T-18 skipped: Linux (GitHub-hosted runner) only" fi +# --- T-20 code that runs inside the isolation uses no here-document or +# here-string (static): on macOS bash 3.2 would need /tmp for them, which +# the sandbox does not allow, so one would fail there (and a missed failure +# can look like a pass). The suites, the scripts they run and bin/devenv; +# the payload is checked by its hash. The one allowed match is the edge +# guard's test text for a Caddyfile here-document. +t20=$(cd "$root" && grep -nE "(^|[^<])<<-?[[:space:]]*[\"'\\\\]?[A-Za-z_]|<<<" -- bin/devenv tests/devenv/unit.sh \ + tests/devenv/tally.sh scripts/ci/no-docker.sh scripts/ci/guard-*.sh scripts/ci/tests/guard-*.test.sh 2>&1 | + grep -vE '^[^:]+:[0-9]+:[[:space:]]*#' | grep -vF 'heredoc (<<) is not supported') +printf '%s\n' "$t20" | grep . | sed 's/^/INFO /' +[ -z "$t20" ] +result "T-20 code run inside the isolation has no here-document or here-string" $? + +# --- T-23 the helpers this suite runs outside any isolation are pinned to +# reviewed hashes in this file (static): none is hashed after it was +# written. On dfb6ee4: canary.py and drive.sh were written with +# here-documents and their sha256 taken afterwards. +grep -qE '^helper_sums="canary\.py=[0-9a-f]{64} drive\.sh=[0-9a-f]{64} payload=[0-9a-f]{64}"$' "$here/no-docker.test.sh" && + ! grep -nE '^[^#]*helper_sums=.*\$\(' "$here/no-docker.test.sh" >/dev/null && + ! grep -nE '^[^#]*>"?\$helpers/' "$here/no-docker.test.sh" >/dev/null +result "T-23 the helpers run outside any isolation are pinned to reviewed sha256 values, not hashed after writing" $? + # --- T-19 CI order (static): no step that is not isolated follows an # isolated one in a job; the required jobs exist; no if: conditions. # On 38b5ebd: "Allocator unit tests" came before the real-Docker steps and diff --git a/scripts/ci/tests/no-docker/canary.py b/scripts/ci/tests/no-docker/canary.py new file mode 100644 index 0000000..c20d378 --- /dev/null +++ b/scripts/ci/tests/no-docker/canary.py @@ -0,0 +1,34 @@ +import json, os, socketserver, sys +path, log = sys.argv[1], sys.argv[2] +class H(socketserver.StreamRequestHandler): + def handle(self): + while True: + line = self.rfile.readline() + if not line: + return + headers = {} + while True: + h = self.rfile.readline() + if h in (b"\r\n", b"\n", b""): + break + k, _, v = h.decode("latin-1").partition(":") + headers[k.strip().lower()] = v.strip() + if headers.get("user-agent") != "no-docker-probe": + with open(log, "a") as f: + f.write(line.decode("latin-1").strip() + " ua=" + headers.get("user-agent", "-") + "\n") + target = line.split()[1].decode("latin-1") if len(line.split()) > 1 else "/" + if target.endswith("/_ping"): + body = b"OK" + elif target.endswith("/version"): + body = json.dumps({"Version": "canary-engine", "ApiVersion": "1.45", "MinAPIVersion": "1.24", + "Os": "linux", "Arch": "amd64"}).encode() + else: + body = json.dumps({"message": "canary-engine"}).encode() + self.wfile.write(b"HTTP/1.1 200 OK\r\nApi-Version: 1.45\r\nOstype: linux\r\n" + b"Content-Type: application/json\r\nContent-Length: %d\r\n\r\n" % len(body)) + if not line.startswith(b"HEAD"): + self.wfile.write(body) + self.wfile.flush() +class S(socketserver.ThreadingMixIn, socketserver.UnixStreamServer): + daemon_threads = True +S(path, H).serve_forever() diff --git a/scripts/ci/tests/no-docker/drive.sh b/scripts/ci/tests/no-docker/drive.sh new file mode 100644 index 0000000..54ea1c2 --- /dev/null +++ b/scripts/ci/tests/no-docker/drive.sh @@ -0,0 +1,16 @@ +canary_py=$1 data=$2 canary=$3 +shift 3 +mkdir -p "$data" || exit 2 +: >"$data/canary.log" || exit 2 +rm -f "$canary" +python3 -I "$canary_py" "$canary" "$data/canary.log" & +cpid=$! +for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do [ -S "$canary" ] && break; sleep 0.2; done +[ -S "$canary" ] || { echo "HARNESS ERROR: the canary engine did not start"; kill "$cpid"; exit 2; } +: >"$data/drive-started" +DOCKER_HOST=unix://$canary "$@" +rc=$? +sleep 0.5 # let the canary finish writing +kill "$cpid" 2>/dev/null +wait "$cpid" 2>/dev/null +exit "$rc" diff --git a/scripts/ci/tests/no-docker/payload b/scripts/ci/tests/no-docker/payload new file mode 100644 index 0000000..0fcfeb3 --- /dev/null +++ b/scripts/ci/tests/no-docker/payload @@ -0,0 +1,37 @@ +qa_log=@LOG@ qa_canary=@CANARY@ qa_cfg=@CFG@ qa_marker=@MARKER@ +qa() { # case command... + qa_n=$1 + shift + qa_out=$("$@" 2>&1) + printf '%s rc=%s out=%s\n' "$qa_n" "$?" "$(printf '%s' "$qa_out" | tr '\n' ' ' | cut -c1-200)" >>"$qa_log" +} +qa_fmt='{{.Server.Version}}' +qa inherited-env docker version --format "$qa_fmt" +qa path-lookup docker -H "unix://$qa_canary" version --format "$qa_fmt" +for qa_p in /usr/local/bin/docker /usr/bin/docker /opt/homebrew/bin/docker /Applications/Docker.app/Contents/Resources/bin/docker; do + [ ! -x "$qa_p" ] || qa "absolute-path:$qa_p" "$qa_p" -H "unix://$qa_canary" version --format "$qa_fmt" +done +qa builtin-command command docker -H "unix://$qa_canary" version --format "$qa_fmt" +qa env-wrapper env docker -H "unix://$qa_canary" version --format "$qa_fmt" +qa env-docker-host env DOCKER_HOST="unix://$qa_canary" docker version --format "$qa_fmt" +qa context env DOCKER_CONFIG="$qa_cfg" docker --context canary version --format "$qa_fmt" +qa nested-bash bash -c 'docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" +qa nested-sh sh -c 'docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" +qa_x=$(docker -H "unix://$qa_canary" version --format "$qa_fmt" 2>&1) +printf 'command-substitution rc=%s out=%s\n' "$?" "$(printf '%s' "$qa_x" | tr '\n' ' ')" >>"$qa_log" +qa_x=`docker -H "unix://$qa_canary" version --format "$qa_fmt" 2>&1` +printf 'backticks rc=%s out=%s\n' "$?" "$(printf '%s' "$qa_x" | tr '\n' ' ')" >>"$qa_log" +qa_dk=docker +qa variable "$qa_dk" -H "unix://$qa_canary" version --format "$qa_fmt" +qa quoted "docker" -H "unix://$qa_canary" version --format "$qa_fmt" +qa exec-subshell sh -c 'exec docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" +qa xargs sh -c 'echo version | xargs docker -H "unix://$1"' _ "$qa_canary" +(docker -H "unix://$qa_canary" version --format "$qa_fmt" >"$qa_log.bg" 2>&1; echo "background rc=$? out=$(tr '\n' ' ' <"$qa_log.bg")" >>"$qa_log") & +wait +qa sudo sudo -n docker -H "unix://$qa_canary" version --format "$qa_fmt" +qa curl-unix-socket curl -sS --max-time 5 --unix-socket "$qa_canary" http://docker/version +qa python-raw-socket python3 -c 'import socket,sys; s=socket.socket(socket.AF_UNIX); s.connect(sys.argv[1]); s.sendall(b"GET /version HTTP/1.0\r\n\r\n"); print(s.recv(4096).decode())' "$qa_canary" +qa real-socket-cli docker -H unix:///var/run/docker.sock version --format "$qa_fmt" +qa real-socket-curl curl -sS --max-time 5 --unix-socket /var/run/docker.sock http://docker/_ping +qa home-write touch "$qa_marker" +exit 97 From d9f41c9ffae4a4e19c4337c20a9f9d687a905475 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 17:08:43 +0100 Subject: [PATCH 22/77] test(ci): reject continue-on-error in the CI isolation order check A job or step with continue-on-error would let a failing step, a security gate included, leave its job green. T-19 now fails on any such key. --- scripts/ci/tests/ci-isolation-order.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/ci/tests/ci-isolation-order.py b/scripts/ci/tests/ci-isolation-order.py index e1030bd..6101845 100644 --- a/scripts/ci/tests/ci-isolation-order.py +++ b/scripts/ci/tests/ci-isolation-order.py @@ -16,7 +16,9 @@ - the required jobs "Repository guard" and "Dev environment" exist and each has an isolated step; - no job or step has an `if:` condition (one could run a non-isolated - step after a failed isolated one, or skip a required check); + step after a failed isolated one, or skip a required check) or a + `continue-on-error:` key (a failed step, a security gate included, + would no longer fail its job); - every tracked file that mentions no-docker.sh is either part of the isolation itself, documentation, the workflow, or has the re-exec stanza in its recognised form, so a suite isolated another way is @@ -71,7 +73,7 @@ def parse(path): if not in_jobs: continue if n == 2: - job = {"name": "", "steps": [], "if": False} + job = {"name": "", "steps": [], "if": False, "coe": False} jobs[line.strip().rstrip(":")] = job step = None elif n == 4 and job is not None: @@ -81,10 +83,12 @@ def parse(path): job["name"] = key[len("name:"):].strip().strip("'\"") if key.startswith("if:"): job["if"] = True + if key.startswith("continue-on-error:"): + job["coe"] = True elif n >= 6 and job is not None: stripped = line.strip() if n == 6 and stripped.startswith("- "): - step = {"name": "", "text": "", "if": False, "run": [], "in_run": False} + step = {"name": "", "text": "", "if": False, "coe": False, "run": [], "in_run": False} job["steps"].append(step) stripped = stripped[2:] n = 8 @@ -96,6 +100,8 @@ def parse(path): step["name"] = stripped[len("name:"):].strip().strip("'\"") elif stripped.startswith("if:"): step["if"] = True + elif stripped.startswith("continue-on-error:"): + step["coe"] = True elif stripped.startswith("run:"): value = stripped[len("run:"):].strip() step["in_run"] = True @@ -204,6 +210,9 @@ def main(): report(required in names, f"required job {required!r} exists") report(not any(j["if"] or any(s["if"] for s in j["steps"]) for j in jobs.values()), "no job or step has an if: condition") + coe = [s["name"] or job["name"] for job in jobs.values() for s in [{"name": ""}] * job["coe"] + job["steps"] + if s.get("coe", job["coe"] and not s["name"])] + report(not coe, "no job or step has continue-on-error" + (f" ({', '.join(repr(c) for c in coe)})" if coe else "")) for jid, job in jobs.items(): first = None late = [] From e8c15980114328ec5641dde40e0d07c67c35fb20 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 17:08:43 +0100 Subject: [PATCH 23/77] docs(dev-environment): describe the macOS process ending, here-documents and debugging limits Where bash 3.2 keeps here-documents, why code run inside uses none, the supervisor and its markers, the measured launchd, terminal, ptrace and task-port limits, the residual risks after SIGKILL of no-docker.sh, the pinned helpers and the new regression cases; the probe's checks are described per platform. --- docs/dev-environment.md | 172 ++++++++++++++++++++++++++++++---------- 1 file changed, 131 insertions(+), 41 deletions(-) diff --git a/docs/dev-environment.md b/docs/dev-environment.md index 93fdf78..59609eb 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -444,9 +444,11 @@ Supported platforms: | other systems | refused (exit 125) | — | | any platform as uid 0 | refused (exit 125) | — | -Each property below is enforced by the sandbox or the kernel and checked -from inside by `scripts/ci/no-docker-probe.py` before the command starts. If -any check fails, a tool is missing, a mount fails, the profile is rejected or +Each property below is enforced by the sandbox profile or the kernel. Before +the command starts, `scripts/ci/no-docker-probe.py` checks from inside the +ones each platform's section lists as checked; on macOS the others rest on +`sandbox-exec` accepting the profile and on the regression tests. If any +check fails, a tool is missing, a mount fails, the profile is rejected or the command was not started inside the isolation, the command does not run and the exit status is 125. @@ -482,10 +484,11 @@ The `sandbox-exec` profile denies: column header or prints a LISTEN row that cannot be parsed, the run is refused. The probe connects to each held port; - file writes anywhere except in the run's directory, `/dev/null`, - `/dev/zero`, `/dev/tty`, `/dev/dtracehelper`, `/dev/fd/*` and the - `sh-thd*` here-document files of bash 3.2, which bash creates in `/tmp` - whatever `$TMPDIR` is. The probe checks that `$HOME`, the user's - temporary directory, `/tmp` and `/private/var/tmp` are not writable and + `/dev/zero`, `/dev/tty`, `/dev/dtracehelper` and `/dev/fd/*`. Nothing in + `/tmp` or `/private/var/tmp` is writable, so no file another process + keeps there can be changed, replaced or created from inside. The probe + checks that `$HOME`, the user's temporary directory, `/tmp`, + `/private/var/tmp` and an `sh-thd-*` file in `/tmp` are not writable and that `$TMPDIR` is. This `/dev` list was measured with bash 3.2, python3, `mktemp`, `/usr/bin/git`, `tests/devenv/unit.sh` and the guard suites. Writing to `/dev/random`, `/dev/urandom` and other terminals' `/dev/ttys*` @@ -495,14 +498,72 @@ The `sandbox-exec` profile denies: - signals to any process outside the sandbox. Processes of the run can signal each other, including process groups. The probe sends signal 0 to `no-docker.sh` outside: EPERM; +- debugging any process (`system-debug`): `ptrace(PT_ATTACHEXC)` fails with + EPERM, even against a process of the user that allows debuggers + (`get-task-allow`). Measured: without this rule the attach succeeds, and + a debugger could then stop, signal or end that process; +- the task ports of processes outside the sandbox (control, inspect, read + and name ports), which debuggers and readers such as `vmmap` and `sample` + use. Measured: without the rule they read the memory map of such a + process from inside; with it they are refused, while they still work on + the command's own processes and `ps` still lists every process; +- reading the devices of other terminals (`/dev/ttys*`), which would let + the command read what is typed in them. The command keeps its own + terminal through `/dev/tty` and the descriptors it inherits; - preference writes through `cfprefsd` and `TIOCSTI` on the terminal; - `open`, `osascript`, `launchctl` and `ssh`, job submission to launchd, - Launch Services and Apple Events; + Launch Services and Apple Events. Measured with a re-signed copy of + `launchctl` inside: `submit` and `bootstrap` of a job and `setenv` in the + user's launchd domain all fail, and no job or variable appears; - lookups of the services that register login items and background tasks (`com.apple.xpc.smd`, `com.apple.backgroundtaskmanagement*`, `com.apple.coreservices.sharedfilelistd*`) and of Docker Desktop's own services (`com.docker.*`, `com.electron.dockerdesktop.*`). +bash 3.2, which is also `/bin/sh`, keeps each here-document and +here-string in a temporary file, ignoring `$TMPDIR`: in `/private/var/tmp`, +or in `/tmp` where `/var/tmp` is not writable, as inside a sandbox. Neither +is writable inside, so a here-document fails there ("cannot create temp +file for here document"). Measured: bash then falls back to the working +directory, so one works only while the working directory is in the run's +directory. The code that runs inside (`bin/devenv`, `tests/devenv/unit.sh`, +the guard suites and the scripts they run) therefore uses none; T-20 +checks this statically. + +When the command ends, every process it left in the sandbox is ended: a +plain background job, a daemon after `setsid` and a double fork, a +`posix_spawn` child in a session of its own, a process group of its own. +`no-docker.sh` hands its process to a supervisor in +`scripts/ci/no-docker-probe.py`, which starts `sandbox-exec`, waits for it +and then ends what is left. It finds those processes by a marker: the +profile denies reading file `a` of a marker directory of the run (and of +one common to every run), in `$TMPDIR` outside the run's directory, and +leaves file `b` next to it alone. From outside, `sandbox_check()` asks how a +process's sandbox would decide; a process is in the run's sandbox exactly +when it is denied `a` and allowed `b`. Measured: an unsandboxed process is +allowed both, other sandboxes decide both the same way, and the files must +exist, or every process is "denied". A process cannot leave its sandbox, +and every process it starts inherits it, however it detaches. The +supervisor sends SIGKILL to each member as soon as it finds one, after +reading its start time and membership again, and lists the processes again +until there is none. A killed process cannot start another one, and +nothing a member does can undo SIGKILL. The supervisor needs no new process +to do this, so a command that uses up the processes it may start cannot +stop it. It prints how many processes it ended. If a member cannot be ended +within 10 seconds (for example one of another user), the run fails with +status 125. SIGINT, SIGTERM, SIGHUP or SIGQUIT delivered to `no-docker.sh` +end the command and everything it left at once, and the status is 128 plus +the signal number; a signal the caller ignores stays ignored. The probe +checks from inside that the sandbox carries both markers. + +SIGKILL of `no-docker.sh` (or a crash of the supervisor) cannot be handled: +what the command left keeps running, still inside the sandbox with every +limit above. Before each run, `no-docker.sh` therefore looks for processes +of the common marker that run on their own (their supervisor is gone) and +refuses to start while there is one, naming them. A concurrent run's own +processes are under its supervisor and do not count, but a daemon it left +does until it ends. + Setuid programs cannot run in the sandbox, so `ps` is an ad-hoc signed copy and `sudo` is unavailable. The U-16 check that another local user cannot read a secret therefore prints INFO and is not counted; the U-16 mode checks @@ -629,33 +690,36 @@ them. #### Accepted residual risks -- **macOS leftover processes.** Background processes of the command can - outlive it. They stay inside the sandbox, with every limit above, but - have an unbounded time for the routes below. Killing the run's process - group or session afterwards is a follow-up. +- **SIGKILL of `no-docker.sh` (macOS).** Its supervisor cannot end what + the command left. Those processes stay in the sandbox, with every limit + above, until someone ends them or the user logs out; meanwhile no new + isolated run starts. Processes left by runs of a version without the + markers (before the supervisor) are not recognised: log out once after + updating. +- **Process identity (macOS).** A process is signalled only after its + start time and membership were read again, microseconds before. A pid + reused within that window by a process outside could still be ended: + an availability risk, not a way out. - **Late listeners.** A loopback port that starts listening after the run started (on macOS) is reachable, as are the ports the command opens - itself. This includes a VM's port forward that starts later, or one - started while a leftover process still runs. The probe asks each - listening port whether it is an engine only at the start. -- **Login items and launchd environment through copies (macOS).** The - services that register login items and background tasks are denied. A - re-signed copy of `launchctl` loses its entitlements. Whether such a copy - can still `setenv` in the user's launchd domain was not measured. -- **Here-document files in `/tmp` (macOS).** bash 3.2, which is also - `/bin/sh`, writes every here-document to `/tmp/sh-thd*` whatever - `$TMPDIR` is. It creates the file, closes it, opens it again by name and - then removes it. Measured: without the `sh-thd*` allowance, here-documents - in the sandbox fail ("cannot create temp file for here document"), also - with `$TMPDIR` in the run's directory. So the allowance stays. Code in - the sandbox can therefore remove and recreate an `sh-thd*` file of a - bash 3.2 that runs outside the sandbox, between that bash's close and - reopen. That injects shell input into the outside script. The window is - short, but a leftover process (above) can wait for it. SBPL cannot limit - the rule to files the sandbox created itself. -- **Terminals and the pasteboard (macOS).** Writes to other terminals' - devices are denied. The pasteboard (`pbcopy`) is not; using it needs a - person to paste. + itself. This includes a VM's port forward that starts later. The probe + asks each listening port whether it is an engine only at the start. The + other way round, a loopback port the command opens during the run can be + reached by programs outside, which may take its answers for those of a + local service (for example a readiness check of `bin/devenv` run at the + same time). +- **Login items and launchd (macOS).** The services that register login + items and background tasks are denied, and launchd jobs and `setenv` + through a re-signed `launchctl` fail (measured). Services that launchd + starts on the command's behalf run outside the sandbox; they are limited + by the lookups and Apple Events denied above, not by the supervisor. +- **Terminals and the pasteboard (macOS).** Reading and writing other + terminals' devices are denied. A process of the command can still make + itself the foreground process group of its own terminal while it runs. + The pasteboard (`pbcopy`) is not denied; using it needs a person to + paste. +- **Reading other processes (macOS).** The environment and arguments of the + user's other processes are readable (`ps eww`), as are the user's files. - **Writes through other processes (macOS).** The file-write limit covers the command's own writes. Writes made for it by system services are limited by the launchd, Launch Services, Apple Events, preference and @@ -707,7 +771,7 @@ private. `no-docker.test.sh`, or a script with the re-exec stanza. A step that mixes such commands with others fails; - the required jobs exist; -- no job or step has an `if:` condition; +- no job or step has an `if:` condition or a `continue-on-error:` key; - every other file that mentions `no-docker.sh` has the recognised re-exec stanza. @@ -732,10 +796,14 @@ request. - Files that code under test could write are read only when they are regular files, so a planted FIFO or link cannot hang or redirect the test. -- **T-14:** the helpers that the test runs outside any isolation, and the - mutant's text, are kept outside every directory that code under test can - write. Their sha256 is checked immediately before each such run. The - case also checks that a changed copy is detected and not run. +- **T-14 and T-23:** the helpers that the test runs outside any isolation + (`canary.py`, `drive.sh`) and the mutant's text are tracked files in + `scripts/ci/tests/no-docker/`. Their sha256 values are constants in the + test, set when they were reviewed, and every copy is checked against + them immediately before each run outside, so a copy changed at any time + is never run. The copies are kept outside every directory that code + under test can write. T-14 checks that a changed copy is detected and not + run, and T-23 that the values are constants. The other cases run plain commands that only print, connect, write markers in the test's directories or send signal 0 or TERM to the test's own @@ -745,15 +813,35 @@ in the test's directories or send signal 0 or TERM to the test's own - **T-01 and T-02:** `--check`, and a claimed but unproven isolation. - **T-07:** loopback ports that were already listening are unreachable, while ports opened inside are reachable. -- **T-06 and T-08 to T-15, macOS only:** - - **T-06 and T-11:** the write limits; +- **T-06, T-08 to T-15, T-21, T-22 and T-24 to T-29, macOS only:** + - **T-06 and T-11:** the write limits, including that a bash 3.2 + here-document fails; - **T-08:** the complete port list; - **T-09 and T-12:** the `TMPDIR` allowlist; - **T-10:** preferences and `TIOCSTI`; - **T-13:** a canary outside cannot be signalled (EPERM) and survives, while the command's own child and process group can be signalled; - **T-14:** integrity of the helpers; - - **T-15:** `--work`, accepted and refused cases. + - **T-15:** `--work`, accepted and refused cases; + - **T-21:** canary files named like bash 3.2 here-document files in + `/tmp` and `/private/var/tmp` cannot be opened for writing, truncated, + removed, renamed, replaced or chmodded, and none can be created; + - **T-22:** a file outside the run's directory cannot be changed through + a hard link, a symbolic link, a rename over it or an exchange; + - **T-24:** four processes the command leaves (a background child, a + `setsid` daemon after a double fork, a `posix_spawn` child in a session + of its own, a process group of its own) are all ended, after a normal + end, a failed command and a command killed by SIGKILL; + - **T-25:** SIGINT, SIGTERM and SIGHUP to `no-docker.sh` end the command + and those four processes at once (status 128 plus the signal); + - **T-26:** after SIGKILL of `no-docker.sh`, its leftovers keep a new run + from starting (125) until they are ended through the run's marker; + - **T-27:** when the probe cannot make the markers, look for leftovers or + supervise, the command is not run; + - **T-28:** a debugger attach from inside to a test canary outside that + allows debuggers fails, while it works from outside; + - **T-29:** another terminal's device cannot be read from inside, while + it can from outside. - **T-16 to T-18, Linux only:** - **T-16:** a canary inside sees only the namespace's processes, cannot read `/proc/1/root` and cannot connect to any engine, session or host @@ -769,6 +857,8 @@ in the test's directories or send signal 0 or TERM to the test's own a workspace outside `$HOME` and a caller without a `Runner.Worker` ancestor are refused. - **T-19:** the static CI order check, on both platforms. +- **T-20:** a static check that the code run inside the isolation has no + here-document or here-string, on both platforms. The macOS cases run only on a developer's Mac: there is no macOS CI job. From 44644e476aeee80bbe9fdfa28c69d8dc0de59914 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 17:28:58 +0100 Subject: [PATCH 24/77] fix(test): rule out leftovers before writing the run directory and keep their marker on failure On macOS the marker directory and the check for processes of an earlier run now come before anything is written into the run's directory, so a leftover cannot prepare files there (such as a link where the ps copy is made). When the supervisor cannot end every process, it keeps the run's marker directory and names the command that ends them. A signal that arrives while the wait for the command is being left can no longer skip ending the processes. --- scripts/ci/no-docker-probe.py | 25 +++++++++++++-------- scripts/ci/no-docker.sh | 42 ++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/scripts/ci/no-docker-probe.py b/scripts/ci/no-docker-probe.py index d9600c7..d0a1dbe 100644 --- a/scripts/ci/no-docker-probe.py +++ b/scripts/ci/no-docker-probe.py @@ -521,15 +521,18 @@ def on_signal(signum, frame): except OSError as e: print(f"no-docker: cannot start the isolation: {e}", file=sys.stderr) try: - # A signal that came while it was being started is not lost: the - # wait is skipped. - waiting[0] = True - if child is not None and not caught: - status = child.wait() + try: + # A signal that came while it was being started is not + # lost: the wait is skipped. + waiting[0] = True + if child is not None and not caught: + status = child.wait() + finally: + waiting[0] = False except Interrupted: + # Also when it arrives in the finally clause above (the handler + # clears waiting before it raises, so it raises only once). pass - finally: - waiting[0] = False try: ended = darwin_end(sbx) if child is not None: @@ -543,13 +546,17 @@ def on_signal(signum, frame): failure = str(e) finally: started = os.path.exists(started_file) - shutil.rmtree(markdir, ignore_errors=True) + # Kept when processes are left, so they can still be ended through + # it (--end MARKDIR). + if not failure: + shutil.rmtree(markdir, ignore_errors=True) if remove != "-": shutil.rmtree(remove, ignore_errors=True) if ended: print(f"no-docker: ended {ended} processes the command left running", file=sys.stderr) if failure: - return refused(f"processes of the isolated command could not all be ended: {failure}") + return refused(f"processes of the isolated command could not all be ended: {failure}; " + f"end them with: python3 -I {os.path.abspath(__file__)} --end {markdir}") if not started: return refused(f"the isolation tool returned {status} without starting the command; the command was not run") if caught: diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh index 1a2ebd7..bab96bd 100755 --- a/scripts/ci/no-docker.sh +++ b/scripts/ci/no-docker.sh @@ -34,10 +34,10 @@ # Events, login item or background task registration and no lookup # of Docker's own services. When the command ends, or no-docker.sh # gets SIGINT, SIGTERM, SIGHUP or SIGQUIT, every process left in the -# sandbox is killed (found by a marker of the run); a process left -# by a run that could not do that (SIGKILL) keeps new runs from -# starting. $TMPDIR must be the per-user directory under -# /private/var/folders. +# sandbox is killed (found by a marker of the run), or the run +# fails (125); a process left by a run that could not do that +# (SIGKILL, failure) keeps new runs from starting. $TMPDIR must be +# the per-user directory under /private/var/folders. # Linux sudo unshare: new network (loopback only), mount, pid, and IPC # namespaces; empty tmpfs over /run, /var/run, /tmp, /var/tmp, # /dev/shm and /sys/fs/cgroup, a fresh /dev/mqueue, every other mount @@ -200,6 +200,24 @@ case $os in work=$(cd -P "$work" && pwd -P) || refuse "cannot resolve the temporary directory" trap 'rm -rf "$work"' EXIT fi + # The sandbox carries a marker of this run and one common to every run + # (a denied read of a file in each marker directory, which are in + # $tmp, outside everything the sandbox may write), by which + # no-docker-probe.py finds its processes from outside: a process cannot + # leave its sandbox, and every process it starts inherits it. A + # process left by an earlier run is ruled out before anything is + # written into the run's directory. + markdir=$(python3 -I "$probe" --markers "$tmp") || refuse "cannot make the marker of the run" + case $markdir in "$tmp"/no-docker-mark.?*) ;; *) refuse "unexpected marker directory $markdir" ;; esac + plain "$markdir" || refuse "unexpected character in the marker directory $markdir" + if [ -n "$given_work" ]; then + remove=- + trap 'rm -rf "$markdir"' EXIT + else + remove=$work + trap 'rm -rf "$work" "$markdir"' EXIT + fi + python3 -I "$probe" --orphans "$tmp" || refuse "cannot rule out a process left by an earlier isolated run" ;; Linux) # Only GitHub-hosted runners: the VM is discarded after the job, sudo @@ -306,22 +324,6 @@ case $os in done envargs+=("NO_DOCKER_HELD_PORTS=$(printf '%s' "$listening" | tr '\n' ' ')" "NO_DOCKER_OUTER_PID=$$" "NO_DOCKER_USER_TMP=$tmp") - # The sandbox carries a marker of this run and one common to every run - # (a denied read of a file in each marker directory, which are in - # $tmp, outside everything the sandbox may write), by which - # no-docker-probe.py finds its processes from outside: a process cannot - # leave its sandbox, and every process it starts inherits it. - markdir=$(python3 -I "$probe" --markers "$tmp") || refuse "cannot make the marker of the run" - case $markdir in "$tmp"/no-docker-mark.?*) ;; *) refuse "unexpected marker directory $markdir" ;; esac - plain "$markdir" || refuse "unexpected character in the marker directory $markdir" - if [ -n "$given_work" ]; then - remove=- - trap 'rm -rf "$markdir"' EXIT - else - remove=$work - trap 'rm -rf "$work" "$markdir"' EXIT - fi - python3 -I "$probe" --orphans "$tmp" || refuse "cannot rule out a process left by an earlier isolated run" envargs+=("NO_DOCKER_MARK=$markdir") # File writes: the run's directory; of /dev only what shells and # interpreters open for writing (measured, docs/dev-environment.md). From e8ed7ed94651769b6aac968011c25fb293c5647f Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 17:28:59 +0100 Subject: [PATCH 25/77] docs(dev-environment): state the failure cases of the macOS process ending and the Linux probe additions A run fails with 125 when processes remain after 10 seconds, for example under a fork storm; after SIGKILL of no-docker.sh or a failed ending the command and its leftovers keep their remaining channels until they are ended. The Linux list names the proc and nsfs check, the full-backlog rule and the uid and gid line, and says the path has not yet run in CI. --- docs/dev-environment.md | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/docs/dev-environment.md b/docs/dev-environment.md index 59609eb..ea077f4 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -549,9 +549,12 @@ reading its start time and membership again, and lists the processes again until there is none. A killed process cannot start another one, and nothing a member does can undo SIGKILL. The supervisor needs no new process to do this, so a command that uses up the processes it may start cannot -stop it. It prints how many processes it ended. If a member cannot be ended -within 10 seconds (for example one of another user), the run fails with -status 125. SIGINT, SIGTERM, SIGHUP or SIGQUIT delivered to `no-docker.sh` +stop it. It prints how many processes it ended. If members are still +there after 10 seconds (one of another user, or members that start new +processes faster than they are ended), the run fails with status 125. The +run's marker directory is then kept, the message names the command that +ends them (`no-docker-probe.py --end `), and until then no +new run starts. SIGINT, SIGTERM, SIGHUP or SIGQUIT delivered to `no-docker.sh` end the command and everything it left at once, and the status is 128 plus the signal number; a signal the caller ignores stays ignored. The probe checks from inside that the sandbox carries both markers. @@ -657,10 +660,18 @@ The probe checks from inside: `/proc/self/mountinfo` and the mount ID of each path. Creating a file in `$HOME`, `$GITHUB_WORKSPACE` and `$RUNNER_TEMP` must fail as well; - the working directory is reachable by its path; +- no other `proc` file system and no `nsfs` mount is reachable by path; - there is no socket in the private directories, and none of the listed - Unix sockets accepts a stream, seqpacket or datagram connection. - Pathname sockets on a read-only mount would still accept one, which is - why every one of them is tried. + Unix sockets accepts a stream, seqpacket or datagram connection (a + listener with a full backlog counts as accepting). Pathname sockets on a + read-only mount would still accept one, which is why every one of them + is tried. + +The probe also prints the uids and gids of the command. The command keeps +the caller's primary group; on a runner that may be the group of the +Docker socket, which is kept away by the hidden path and the connect test +above. This Linux path has not yet run in CI in this version: the first +run on a GitHub-hosted runner is its first evidence. The output between the start and the end of the isolated command is framed with `::stop-commands::`, so the command cannot issue @@ -690,12 +701,15 @@ them. #### Accepted residual risks -- **SIGKILL of `no-docker.sh` (macOS).** Its supervisor cannot end what - the command left. Those processes stay in the sandbox, with every limit - above, until someone ends them or the user logs out; meanwhile no new - isolated run starts. Processes left by runs of a version without the - markers (before the supervisor) are not recognised: log out once after - updating. +- **SIGKILL of `no-docker.sh` (macOS), or a failed ending.** Its + supervisor cannot end the command or what it left. They keep running in + the sandbox, with every limit above, until someone ends them + (`no-docker-probe.py --end `, or by pid); meanwhile no + new isolated run starts. Until then the channels the sandbox leaves open + (their own terminal, loopback ports, the pasteboard, reading the user's + files and process information) stay available to them, without a time + limit. Processes left by runs of a version without the markers (before + the supervisor) are not recognised; end them by pid after updating. - **Process identity (macOS).** A process is signalled only after its start time and membership were read again, microseconds before. A pid reused within that window by a process outside could still be ended: From 2b16025ead5fc45cc1e3dc56efd90d116345aaad Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 18:51:10 +0100 Subject: [PATCH 26/77] fix(test): report a macOS run done only when it is shown that none of its processes runs One process table without a member of the run ended the run as a success, so a process that keeps forking and ending could survive it (B-1). The supervisor now takes one copy of the kernel's process list at a time and needs a copy in which every process is shown, by its unique id read before and after the check, not to be one of the run's, or was dead before that copy; a process that ended before it could be checked leaves the copy open. Members found are killed at once. Within 10 s otherwise the run fails with 125 and keeps its marker directory. The marker directory records the run's owner; before a run, directories whose owner is gone are checked the same way without killing, removed once shown empty, and otherwise refuse the run. --end refuses a run whose owner still runs. A process that ends while it is checked is no longer a false 125, and the supervisor's own child is killed on any failure. The marker common to every run is no longer needed. --- scripts/ci/no-docker-probe.py | 478 ++++++++++++++++++++++------------ scripts/ci/no-docker.sh | 31 ++- 2 files changed, 328 insertions(+), 181 deletions(-) diff --git a/scripts/ci/no-docker-probe.py b/scripts/ci/no-docker-probe.py index d0a1dbe..406a071 100644 --- a/scripts/ci/no-docker-probe.py +++ b/scripts/ci/no-docker-probe.py @@ -48,17 +48,19 @@ --seal WORK as root in the new mount namespace: make every mount reachable by path read-only, except the private file systems and WORK, and verify it (Linux); - --markers TMP make sure the marker directory common to every run exists - in the per-user temporary directory TMP, make one for a new - run and print its path (macOS); - --orphans TMP fail if a process of an earlier run is still alive on its - own (macOS); + --markers TMP OWNER + make a marker directory for a new run in the per-user + temporary directory TMP, recording the process OWNER that + supervises it, and print its path (macOS); + --orphans TMP fail if a run that ended without its supervisor may still + have processes (macOS); --supervise MARKDIR STARTED REMOVE -- COMMAND... run COMMAND (sandbox-exec), then end every process left in the sandbox of MARKDIR; signals end it at once; 125 unless the file STARTED exists; removes MARKDIR and the directory REMOVE ("-": none) (macOS, no-docker.sh's last step); - --end MARKDIR end every process of the sandbox of MARKDIR (macOS). + --end MARKDIR end every process of the sandbox of MARKDIR, and remove it + once that is shown (macOS). """ import concurrent.futures @@ -249,7 +251,7 @@ def check_mechanism_darwin(): + ("" if e is not None else ": written")) # The markers by which no-docker.sh finds this sandbox's processes from # outside: reading A is denied, reading B is not. - for d in (os.environ.get("NO_DOCKER_MARK", ""), os.path.join(user_tmp, COMMON_MARK) if user_tmp else ""): + for d in (os.environ.get("NO_DOCKER_MARK", ""),): result = "unknown" if d: try: @@ -266,29 +268,37 @@ def check_mechanism_darwin(): # The processes of one sandbox, seen from outside it. no-docker.sh (and the # suite's own guard) add to the profile `(deny file-read-data (literal A))` -# for the file A of the run's marker directory and of the directory common -# to every run, and leave the file B next to it alone; no other profile -# names these files. sandbox_check() asks the kernel how a process's -# sandbox would decide, so a process is in the run's sandbox exactly when -# it is denied A and allowed B (measured: an unsandboxed process is -# allowed both, other sandboxes deny or allow both; the files must exist, -# or every process is "denied"). A process cannot leave its sandbox, and -# every process it starts inherits it. The marker directories are in the -# per-user temporary directory, outside every directory the sandbox may -# write, so the command cannot remove or replace them. -COMMON_MARK = "no-docker-mark" +# for file A of the run's marker directory and leave file B next to it +# alone; no other profile names these files. sandbox_check() asks the +# kernel how a process's sandbox would decide, so a process is in the +# run's sandbox exactly when it is sandboxed at all, denied A and allowed +# B (measured: an unsandboxed process is allowed both; the files must +# exist, or every process is "denied"). A process cannot leave its +# sandbox, and every process it starts inherits it. The marker directory +# is in the per-user temporary directory, outside every directory the +# sandbox may write, so the command cannot remove or replace it; it also +# records the run's owner (the supervisor), so a run that ended without +# its supervisor can be recognised later. +MARK_PREFIX = "no-docker-mark." SZOMB = 5 -class ProcBsdInfo(ctypes.Structure): - _fields_ = [("flags", ctypes.c_uint32), ("status", ctypes.c_uint32), ("xstatus", ctypes.c_uint32), - ("pid", ctypes.c_uint32), ("ppid", ctypes.c_uint32), ("uid", ctypes.c_uint32), - ("gid", ctypes.c_uint32), ("ruid", ctypes.c_uint32), ("rgid", ctypes.c_uint32), - ("svuid", ctypes.c_uint32), ("svgid", ctypes.c_uint32), ("rfu_1", ctypes.c_uint32), - ("comm", ctypes.c_char * 16), ("name", ctypes.c_char * 32), ("nfiles", ctypes.c_uint32), - ("pgid", ctypes.c_uint32), ("pjobc", ctypes.c_uint32), ("e_tdev", ctypes.c_uint32), - ("e_tpgid", ctypes.c_uint32), ("nice", ctypes.c_int32), ("start_tvsec", ctypes.c_uint64), - ("start_tvusec", ctypes.c_uint64)] +class ProcShortInfo(ctypes.Structure): + # struct proc_bsdshortinfo (PROC_PIDT_SHORTBSDINFO): answers for + # processes of other users too. + _fields_ = [("pid", ctypes.c_uint32), ("ppid", ctypes.c_uint32), ("pgid", ctypes.c_uint32), + ("status", ctypes.c_uint32), ("comm", ctypes.c_char * 16), ("flags", ctypes.c_uint32), + ("uid", ctypes.c_uint32), ("gid", ctypes.c_uint32), ("ruid", ctypes.c_uint32), + ("rgid", ctypes.c_uint32), ("svuid", ctypes.c_uint32), ("svgid", ctypes.c_uint32), + ("rfu", ctypes.c_uint32)] + + +class ProcUniqIdentifierInfo(ctypes.Structure): + # struct proc_uniqidentifierinfo (PROC_PIDUNIQIDENTIFIERINFO): an id + # the kernel never reuses while it runs. + _fields_ = [("uuid", ctypes.c_uint8 * 16), ("uniqueid", ctypes.c_uint64), ("puniqueid", ctypes.c_uint64), + ("idversion", ctypes.c_int32), ("orig_ppidversion", ctypes.c_uint32), + ("reserve3", ctypes.c_uint64), ("reserve4", ctypes.c_uint64)] def private_dir(path): @@ -297,45 +307,91 @@ def private_dir(path): return stat.S_ISDIR(st.st_mode) and st.st_uid == os.getuid() and stat.S_IMODE(st.st_mode) == 0o700 -def marker_files(directory, create): +def regular_file(path): + st = os.lstat(path) + return stat.S_ISREG(st.st_mode) and st.st_uid == os.getuid() + + +def marker_files(directory): """The marker files A and B of directory, which must be a private - directory with two empty regular files of ours; created if asked.""" - if create: - try: - os.mkdir(directory, 0o700) - except FileExistsError: - pass + directory with two empty regular files of ours.""" if not private_dir(directory): raise OSError(f"{directory} is not a directory of ours with mode 0700") files = [] for name in ("a", "b"): path = os.path.join(directory, name) - if create: - try: - os.close(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)) - except FileExistsError: - pass - st = os.lstat(path) - if not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid(): + if not regular_file(path): raise OSError(f"{path} is not a regular file of ours") files.append(os.fsencode(path)) return files -class Sandboxes: - """sandbox_check() and the process table (libproc). Raises OSError if - either is unavailable or gives an answer it cannot interpret, so every - caller fails closed.""" +class Gone(Exception): + """The process ended (or its pid now names another process).""" + + +class ProcTable: + """The process table (libproc). Raises OSError if it is unavailable or + gives an answer that cannot be interpreted, so every caller fails + closed.""" + + PROC_PIDT_SHORTBSDINFO = 13 + PROC_PIDUNIQIDENTIFIERINFO = 17 + + def __init__(self): + try: + self.proc = ctypes.CDLL("/usr/lib/libproc.dylib", use_errno=True) + except OSError as e: + raise OSError(f"libproc is not available: {e}") from None + self.proc.proc_listallpids.argtypes = [ctypes.c_void_p, ctypes.c_int] + self.proc.proc_pidinfo.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64, ctypes.c_void_p, ctypes.c_int] + + def snapshot(self): + """Every pid, from one copy of the kernel's process lists (taken + under one lock, so it holds every process that existed at that + instant, zombies included). A full buffer may mean a cut list, so + it is taken again with a larger one.""" + for _ in range(8): + n = self.proc.proc_listallpids(None, 0) + if n <= 0: + raise OSError("cannot list the processes") + room = n + 4096 + buf = (ctypes.c_int * room)() + got = self.proc.proc_listallpids(buf, ctypes.sizeof(buf)) + if got <= 0: + raise OSError("cannot list the processes") + if got < room: + return [buf[i] for i in range(got) if buf[i] > 0 and buf[i] != os.getpid()] + raise OSError("the process list did not fit") + + def _info(self, pid, flavor, struct): + ctypes.set_errno(0) + n = self.proc.proc_pidinfo(pid, flavor, 0, ctypes.byref(struct), ctypes.sizeof(struct)) + if n == ctypes.sizeof(struct): + return True + if ctypes.get_errno() in (errno.ESRCH, 0): + return False + raise OSError(f"cannot read process {pid} ({os.strerror(ctypes.get_errno())})") + + def identity(self, pid): + """The kernel's unique id of the process, or None if it is gone.""" + u = ProcUniqIdentifierInfo() + return u.uniqueid if self._info(pid, self.PROC_PIDUNIQIDENTIFIERINFO, u) else None + + +class Sandboxes(ProcTable): + """The process table and sandbox_check() for the marker of one run. + Raises OSError if either is unavailable or gives an answer it cannot + interpret, so every caller fails closed.""" - PROC_PIDTBSDINFO = 3 FILTER_PATH = 1 def __init__(self, directory): + super().__init__() self.directory = directory - self.deny, self.allow = marker_files(directory, create=False) + self.deny, self.allow = marker_files(directory) try: sb = ctypes.CDLL("/usr/lib/system/libsystem_sandbox.dylib", use_errno=True) - self.proc = ctypes.CDLL("/usr/lib/libproc.dylib", use_errno=True) self.check_fn = sb.sandbox_check no_report = ctypes.c_int.in_dll(sb, "SANDBOX_CHECK_NO_REPORT").value except (OSError, AttributeError, ValueError) as e: @@ -346,16 +402,16 @@ def __init__(self, directory): self.check_fn.restype = ctypes.c_int self.check_fn.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int] self.kind = self.FILTER_PATH | no_report - self.proc.proc_listallpids.argtypes = [ctypes.c_void_p, ctypes.c_int] - self.proc.proc_pidinfo.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64, ctypes.c_void_p, ctypes.c_int] self.verify() + if self.identity(os.getpid()) is None: + raise OSError("process identities cannot be read") def verify(self): """The marker files still exist and this (unsandboxed) process is allowed both and is not sandboxed. A missing file reads as denied - for every process, so this runs before every scan and every signal; - anything else means the calls do not work as measured.""" - marker_files(self.directory, create=False) + for every process, so this runs before every scan, after every + membership answer and before every signal.""" + marker_files(self.directory) me = os.getpid() if (self._check(me, self.deny), self._check(me, self.allow), self.check_fn(me, None, 0)) != (0, 0, 0): raise OSError("sandbox_check() does not answer as expected for this process") @@ -363,112 +419,187 @@ def verify(self): def _check(self, pid, path): return self.check_fn(pid, b"file-read-data", self.kind, ctypes.c_char_p(path)) - def member(self, pid): - """True or False; raises OSError if it cannot be told. A member must - be sandboxed at all, be denied A and be allowed B.""" + def classify(self, pid): + """("member" | "other" | "zombie", unique id) of pid; raises Gone if + it ended meanwhile, OSError if it cannot be told. Membership is read + between two identity reads, so the answer belongs to one process.""" + first = self.identity(pid) + if first is None: + raise Gone() + s = ProcShortInfo() + if not self._info(pid, self.PROC_PIDT_SHORTBSDINFO, s): + raise Gone() + if s.status == SZOMB: + return "zombie", first sandboxed = self.check_fn(pid, None, 0) a, b = self._check(pid, self.deny), self._check(pid, self.allow) + self.verify() + if self.identity(pid) != first: + raise Gone() if sandboxed not in (0, 1) or a not in (0, 1) or b not in (0, 1): raise OSError(f"sandbox_check() failed for pid {pid} ({sandboxed}, {a}, {b})") - return sandboxed == 1 and a == 1 and b == 0 - - def pids(self): - n = self.proc.proc_listallpids(None, 0) - if n <= 0: - raise OSError("cannot list the processes") - buf = (ctypes.c_int * (n + 1024))() - n = self.proc.proc_listallpids(buf, ctypes.sizeof(buf)) - if n <= 0: - raise OSError("cannot list the processes") - return [buf[i] for i in range(n) if buf[i] > 0 and buf[i] != os.getpid()] - - def info(self, pid): - """(ppid, uid, status, start time) of pid, or None if it is gone.""" - i = ProcBsdInfo() - if self.proc.proc_pidinfo(pid, self.PROC_PIDTBSDINFO, 0, ctypes.byref(i), ctypes.sizeof(i)) != ctypes.sizeof(i): - return None - return i.ppid, i.uid, i.status, (i.start_tvsec, i.start_tvusec) - - def members(self): - """{pid: info} of the members that still run (a zombie has ended).""" - self.verify() - found = {} - for pid in self.pids(): - inf = self.info(pid) - if inf is None or inf[2] == SZOMB: + return ("member" if sandboxed == 1 and a == 1 and b == 0 else "other"), first + + +def sweep(sbx, seconds, kill): + """Show that no process of the sandbox runs any more, killing each one + found if kill (SIGKILL, after its identity and the markers were read + again). Returns (ended, members) once a round proves it: in one + snapshot every pid is a process that is not a member (membership never + changes), or one already dead (a zombie or gone) before that snapshot + was taken. Every process started later descends from a process in the + snapshot (a child is listed before it can run, its parent until it is + reaped), so none of them can be a member either. A pid that ended + before it could be told apart, or became a zombie only now, leaves the + round open: it may have started a member before it died. Raises + OSError when no round proves it within seconds (for example a process + that keeps starting new ones and ending); the caller fails closed.""" + deadline = time.monotonic() + seconds + others = {} # pid: unique id of a process that is not a member + dead = set() # pids seen dead before the current snapshot + ended = 0 + members = set() + rounds = 0 + while True: + rounds += 1 + sbx.verify() + snap = sbx.snapshot() + present = set(snap) + proven = True + dead_now = set() + for pid in snap: + if pid in dead: continue try: - if self.member(pid): - found[pid] = inf - except OSError: - if self.info(pid) is not None: - raise - return found - - -def darwin_markers(tmp): - """For no-docker.sh and the suite: make sure the common marker directory - exists in the per-user temporary directory tmp, make a new one for the - run, and print the run's.""" - marker_files(os.path.join(tmp, COMMON_MARK), create=True) - run = tempfile.mkdtemp(prefix=COMMON_MARK + ".", dir=tmp) - marker_files(run, create=True) + kind, uid = sbx.classify(pid) + except Gone: + if pid not in others: + proven = False + continue + if kind == "other": + others[pid] = uid + elif kind == "zombie": + dead_now.add(pid) + if pid not in others: + proven = False + else: + proven = False + members.add(pid) + if kill and sbx.identity(pid) == uid: + sbx.verify() + try: + os.kill(pid, signal.SIGKILL) + ended += 1 + except ProcessLookupError: + pass + except PermissionError: + pass # another user's (a setuid program): still found next round + if proven: + return ended, members + # A pid absent from a snapshot may name a new process later. + dead = (dead | dead_now) & present + others = {p: u for p, u in others.items() if p in present} + if time.monotonic() > deadline: + live = sorted(p for p in members if sbx.identity(p) is not None) + raise OSError(f"cannot show within {seconds:.0f}s that every process of the isolation ended " + f"({rounds} rounds; still running: {' '.join(map(str, live)) or 'none seen'})") + time.sleep(0.005) + + +def owner_alive(table, directory): + """True if the owner the marker directory records still runs (same pid + and unique id), False if it is gone; raises OSError if the record is + unreadable.""" + rec = os.path.join(directory, "owner") + if not regular_file(rec) or os.lstat(rec).st_size > 64: + raise OSError(f"{rec} is not a regular file of ours") + with open(rec) as f: + fields = f.read().split() + if len(fields) != 2 or not fields[0].isdigit(): + raise OSError(f"{rec} is not an owner record") + return str(table.identity(int(fields[0]))) == fields[1] + + +def darwin_markers(tmp, owner): + """For no-docker.sh and the suite: make a marker directory for a new run + in the per-user temporary directory tmp, recording its owner (pid and + unique id; the supervisor that owner becomes, or the suite), and print + its path.""" + run = tempfile.mkdtemp(prefix=MARK_PREFIX, dir=tmp) + for name in ("a", "b"): + os.close(os.open(os.path.join(run, name), os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)) + table = ProcTable() + uid = table.identity(int(owner)) if owner.isdigit() else None + if uid is None: + shutil.rmtree(run, ignore_errors=True) + raise OSError(f"the owner {owner} of the run does not exist") + # Complete or absent for a reader: written aside, then renamed. + part = os.path.join(run, "owner.part") + fd = os.open(part, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as f: + f.write(f"{owner} {uid}\n") + os.rename(part, os.path.join(run, "owner")) print(run) return 0 -def darwin_orphans(tmp): - """Before a sandbox starts: fail if a process of an earlier run is still - alive on its own, that is a member of the common marker whose topmost - member ancestor was handed to launchd (no live supervisor above it). A - run in progress elsewhere keeps its members under its supervisor; its - daemons count as orphans, so a concurrent run that leaves one blocks - new ones until it ends.""" - sbx = Sandboxes(os.path.join(tmp, COMMON_MARK)) - found = sbx.members() - orphans = [] - for pid, inf in found.items(): - top, seen = inf, 0 - while top[0] in found and seen < 4096: - top = found[top[0]] - seen += 1 - if top[0] <= 1: - orphans.append(pid) - if orphans: - print("no-docker: processes of an earlier isolated run are still running (pids " - + " ".join(str(p) for p in sorted(orphans)) + "); end them first", file=sys.stderr) +def darwin_orphans(tmp, seconds=5.0): + """Before a sandbox starts: refuse if an earlier run ended without its + supervisor (no-docker.sh killed, or its ending failed) and processes of + it may still run. Each run's marker directory records its owner; one + whose owner is gone is checked with sweep(), without killing: proven + empty, it is removed; otherwise (members found, or not provable in + time) the run is refused and they are named. Directories of runs whose + owner lives (other runs in progress) are left alone.""" + table = ProcTable() + stale = [] + for name in sorted(os.listdir(tmp)): + if not name.startswith(MARK_PREFIX): + continue + d = os.path.join(tmp, name) + try: + if not private_dir(d): + continue + if not os.path.lexists(os.path.join(d, "owner")): + # Renamed into place right after the directory is made; an + # old one without it is a run of an earlier version. + if time.time() - os.lstat(d).st_mtime < 60: + continue + stale.append(d) + continue + if not owner_alive(table, d): + stale.append(d) + except FileNotFoundError: + continue # removed meanwhile by its own run + except OSError: + stale.append(d) + left = [] + for d in stale: + if not os.path.lexists(d): + continue # removed meanwhile by its own run + try: + _, members = sweep(Sandboxes(d), seconds, kill=False) + except OSError as e: + # Among others: a marker file is missing, so its processes + # cannot be found any more; the directory stays until someone + # who knows that none runs removes it. + left.append(f"{d}: {e}") + continue + if members: + left.append(f"{d}: processes {' '.join(map(str, sorted(members)))}") + else: + shutil.rmtree(d, ignore_errors=True) + if left: + print("no-docker: an earlier isolated run ended without its supervisor and may have processes left; " + "end them with: python3 -I " + os.path.abspath(__file__) + " --end : " + + "; ".join(left), file=sys.stderr) return 1 return 0 def darwin_end(sbx, seconds=10.0): - """End every process of the run's sandbox: SIGKILL each member as soon as - it is found (its start time and membership read again right before the - signal, so a reused pid is not hit), and list the table again until it - has none. A killed process cannot start another, and nothing a member - does can undo SIGKILL. Needs no new process. Returns the number of - processes ended; raises OSError if members are left after the time - limit (another user's, such as a sudo inside, cannot be signalled).""" - deadline = time.monotonic() + seconds - ended = 0 - while True: - found = sbx.members() - if not found: - return ended - if time.monotonic() > deadline: - raise OSError("processes of the isolation survive (pids " - + " ".join(f"{p}:uid{inf[1]}" for p, inf in sorted(found.items())) + ")") - for pid, inf in found.items(): - now = sbx.info(pid) - sbx.verify() - if now is None or now[3] != inf[3] or not sbx.member(pid): - continue - try: - os.kill(pid, signal.SIGKILL) - ended += 1 - except (ProcessLookupError, PermissionError): - pass - time.sleep(0.01) + """End every process of the run's sandbox (sweep with kill).""" + return sweep(sbx, seconds, kill=True)[0] class Interrupted(Exception): @@ -481,11 +612,15 @@ def darwin_supervise(markdir, started_file, remove, argv): its sandbox; SIGINT, SIGTERM, SIGHUP or SIGQUIT end it at once the same way. Everything that needs a library is loaded before the command starts, and ending it needs no new process, so a command that uses up - the processes it may start cannot prevent it. Removes the run's marker - directory and the directory remove (if not "-"). The status is the - command's, or 125 if the isolation could not be applied (started_file, - which the command's start creates, is missing) or its processes could - not all be ended; after a signal, 128 plus its number.""" + the processes it may start cannot prevent it. The run succeeds only + when a round of sweep() proved that none of its processes runs any + more. Removes the run's marker directory (kept when that could not be + shown, so that the next run refuses and the processes can still be + ended through it) and the directory remove (if not "-"). The status is + the command's, or 125 if the isolation could not be applied + (started_file, which the command's start creates, is missing) or its + processes could not be shown to have ended; after a signal, 128 plus + its number.""" def refused(why): print(f"no-docker: refused: {why}", file=sys.stderr) return 125 @@ -533,21 +668,21 @@ def on_signal(signum, frame): # Also when it arrives in the finally clause above (the handler # clears waiting before it raises, so it raises only once). pass - try: - ended = darwin_end(sbx) - if child is not None: - if child.poll() is None: - # Interrupted before sandbox-exec applied the profile: - # our own child, not yet reaped, so its pid is its own. - child.kill() - status = child.wait(timeout=10) - ended += darwin_end(sbx) - except (OSError, subprocess.TimeoutExpired) as e: - failure = str(e) + if child is not None: + if child.poll() is None: + # Interrupted: our own child, not yet reaped, so its pid is + # still its own (also when sandbox-exec has not applied the + # profile yet). + child.kill() + status = child.wait() + ended = darwin_end(sbx) + except BaseException as e: # noqa: BLE001 - every failure fails closed + failure = str(e) or type(e).__name__ finally: + if child is not None and child.poll() is None: + child.kill() + child.wait() started = os.path.exists(started_file) - # Kept when processes are left, so they can still be ended through - # it (--end MARKDIR). if not failure: shutil.rmtree(markdir, ignore_errors=True) if remove != "-": @@ -555,7 +690,7 @@ def on_signal(signum, frame): if ended: print(f"no-docker: ended {ended} processes the command left running", file=sys.stderr) if failure: - return refused(f"processes of the isolated command could not all be ended: {failure}; " + return refused(f"the processes of the isolated command could not be shown to have ended: {failure}; " f"end them with: python3 -I {os.path.abspath(__file__)} --end {markdir}") if not started: return refused(f"the isolation tool returned {status} without starting the command; the command was not run") @@ -960,14 +1095,23 @@ def main(): sys.stdout.buffer.write(b"".join(p + b"\0" for p in paths)) print(f"no-docker: {len(paths)} Unix sockets listed on {points} mounts in {took:.1f}s", file=sys.stderr) return 0 - if sys.platform == "darwin" and sys.argv[1:2] in (["--markers"], ["--orphans"], ["--end"]) and len(sys.argv) == 3: + if sys.platform == "darwin" and (sys.argv[1:2] == ["--markers"] and len(sys.argv) == 4 + or sys.argv[1:2] in (["--orphans"], ["--end"]) and len(sys.argv) == 3): try: if sys.argv[1] == "--markers": - return darwin_markers(sys.argv[2]) + return darwin_markers(sys.argv[2], sys.argv[3]) if sys.argv[1] == "--orphans": return darwin_orphans(sys.argv[2]) - n = darwin_end(Sandboxes(sys.argv[2])) - print(f"no-docker: ended {n} processes", file=sys.stderr) + sbx = Sandboxes(sys.argv[2]) + try: + alive = owner_alive(sbx, sys.argv[2]) + except FileNotFoundError: + alive = False + if alive: + raise OSError(f"the run of {sys.argv[2]} is still in progress (its owner runs); not ending it") + n = darwin_end(sbx) + shutil.rmtree(sys.argv[2], ignore_errors=True) + print(f"no-docker: ended {n} processes; none is left", file=sys.stderr) return 0 except OSError as e: print(f"no-docker: cannot check the processes of the isolation: {e}", file=sys.stderr) diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh index bab96bd..eaaa020 100755 --- a/scripts/ci/no-docker.sh +++ b/scripts/ci/no-docker.sh @@ -33,11 +33,13 @@ # osascript, launchctl, ssh, launchd job, Launch Services, Apple # Events, login item or background task registration and no lookup # of Docker's own services. When the command ends, or no-docker.sh -# gets SIGINT, SIGTERM, SIGHUP or SIGQUIT, every process left in the -# sandbox is killed (found by a marker of the run), or the run -# fails (125); a process left by a run that could not do that -# (SIGKILL, failure) keeps new runs from starting. $TMPDIR must be -# the per-user directory under /private/var/folders. +# gets SIGINT, SIGTERM, SIGHUP or SIGQUIT, the processes left in +# the sandbox (found by a marker of the run) are killed; the run +# is reported done only when it is shown that none of them runs +# any more, otherwise it fails (125). A run that ended so, or +# without its supervisor (SIGKILL), keeps new runs from starting +# until that is shown. $TMPDIR must be the per-user directory +# under /private/var/folders. # Linux sudo unshare: new network (loopback only), mount, pid, and IPC # namespaces; empty tmpfs over /run, /var/run, /tmp, /var/tmp, # /dev/shm and /sys/fs/cgroup, a fresh /dev/mqueue, every other mount @@ -200,14 +202,15 @@ case $os in work=$(cd -P "$work" && pwd -P) || refuse "cannot resolve the temporary directory" trap 'rm -rf "$work"' EXIT fi - # The sandbox carries a marker of this run and one common to every run - # (a denied read of a file in each marker directory, which are in - # $tmp, outside everything the sandbox may write), by which - # no-docker-probe.py finds its processes from outside: a process cannot - # leave its sandbox, and every process it starts inherits it. A - # process left by an earlier run is ruled out before anything is - # written into the run's directory. - markdir=$(python3 -I "$probe" --markers "$tmp") || refuse "cannot make the marker of the run" + # The sandbox carries a marker of this run (a denied read of a file in + # its marker directory, which is in $tmp, outside everything the + # sandbox may write), by which no-docker-probe.py finds its processes + # from outside: a process cannot leave its sandbox, and every process it + # starts inherits it. The directory records this process as the run's + # owner (the supervisor it becomes). An earlier run that ended without + # its supervisor and may have processes left is ruled out before + # anything is written into the run's directory. + markdir=$(python3 -I "$probe" --markers "$tmp" "$$") || refuse "cannot make the marker of the run" case $markdir in "$tmp"/no-docker-mark.?*) ;; *) refuse "unexpected marker directory $markdir" ;; esac plain "$markdir" || refuse "unexpected character in the marker directory $markdir" if [ -n "$given_work" ]; then @@ -342,7 +345,7 @@ case $os in (deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")$held) (deny file-write*) (allow file-write* (subpath \"$work\") (literal \"/dev/null\") (literal \"/dev/zero\") (literal \"/dev/tty\") (literal \"/dev/dtracehelper\") (regex #\"^/dev/fd/[0-9]+\$\")) -(deny file-read-data (literal \"$tmp/no-docker-mark/a\") (literal \"$markdir/a\")) +(deny file-read-data (literal \"$markdir/a\")) (deny file-read* (regex #\"^/dev/ttys[0-9]+\$\")) (deny user-preference-write) (deny file-ioctl (ioctl-command #x80017472)) From f2288f022da935e91058c9778ef34c158b295e61 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 18:51:10 +0100 Subject: [PATCH 27/77] test(ci): cover pid hopping, concurrent runs and failed endings of the macOS isolation T-30 runs a bounded chain of forks that end at once and checks that the run is never reported done while it hops, that new runs refuse meanwhile and start after it ended. T-31 runs two isolations at the same time, T-32 keeps unrelated processes starting and ending during the ending, T-33 makes the ending impossible to show (125, own child ended, marker kept) and T-34 sends TERM at several moments while processes are started. T-21 creates its canaries in the shared temporary directories with random names, exclusively and without following links, and reads and removes them only while they are the files it made. --- scripts/ci/tests/no-docker.test.sh | 289 +++++++++++++++++++++++++++-- 1 file changed, 276 insertions(+), 13 deletions(-) diff --git a/scripts/ci/tests/no-docker.test.sh b/scripts/ci/tests/no-docker.test.sh index ed8dcc0..1c8021f 100755 --- a/scripts/ci/tests/no-docker.test.sh +++ b/scripts/ci/tests/no-docker.test.sh @@ -43,14 +43,16 @@ # sleep and must be ended by no-docker.sh (any left over are killed by # pid only while their arguments carry the case's marker), attach a # debugger to or read the memory map of this suite's own canary copies -# of /bin/sleep, read a pseudo-terminal of their own, or sleep. +# of /bin/sleep, read a pseudo-terminal of their own, run a bounded +# chain of forks of their own inside the isolation (T-30), remove the +# marker file of a run of their own (T-33), or sleep. # This suite itself runs no docker command. # -# T-06, T-08 to T-15, T-21, T-22 and T-24 to T-29 are macOS only, T-16 to +# T-06, T-08 to T-15, T-21, T-22 and T-24 to T-34 are macOS only, T-16 to # T-18 Linux only (each prints INFO on the other platform); T-19, T-20 and # T-23 are static and run on both. Each case says how it fails against the -# no-docker.sh of 38b5ebd or dfb6ee4, which these cases were written to -# catch. +# no-docker.sh of 38b5ebd, dfb6ee4 or e8ed7ed, which these cases were +# written to catch. # # Usage: no-docker.test.sh # HARNESS= checks another copy of tests/devenv/unit.sh instead of the @@ -109,6 +111,9 @@ tmp_marker=/private/tmp/.flowcrm-no-docker-test-$nonce t11_marker=$tmp/.flowcrm-no-docker-test-t11-$nonce cleanup() { rm -f "$home_marker" "$outside_marker" "$tmp_marker" "$t11_marker" + # T-21's canaries in the shared directories, only while they are ours. + [ -z "${t21a_ino-}" ] || python3 -I -c "$t21_file" remove "$t21a" "$t21a_dev" "$t21a_ino" + [ -z "${t21b_ino-}" ] || python3 -I -c "$t21_file" remove "$t21b" "$t21b_dev" "$t21b_ino" rm -rf "$work" "$ro" "$vt" } trap cleanup EXIT @@ -275,7 +280,7 @@ run_case() { case $port in *[!0-9]* | "") echo "HARNESS ERROR: unexpected port $port" >&2; exit 2 ;; esac held="$held (remote ip \"localhost:$port\")" done - gmark=$(python3 -I "$root/scripts/ci/no-docker-probe.py" --markers "$ndtmp") || exit 2 + gmark=$(python3 -I "$root/scripts/ci/no-docker-probe.py" --markers "$ndtmp" "$$") || exit 2 case $gmark in "$ndtmp"/no-docker-mark.?*) ;; *) echo "HARNESS ERROR: unexpected marker directory $gmark" >&2; exit 2 ;; esac python3 -I "$root/scripts/ci/no-docker-probe.py" --supervise "$gmark" "$data/drive-started" - -- /usr/bin/sandbox-exec -p "(version 1) (allow default) @@ -286,7 +291,7 @@ run_case() { (deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")$held) (deny file-write*) (allow file-write* (subpath \"$data\") (subpath \"$w\") (literal \"/dev/null\") (literal \"/dev/zero\") (literal \"/dev/tty\") (literal \"/dev/dtracehelper\") (regex #\"^/dev/fd/[0-9]+\$\")) -(deny file-read-data (literal \"$ndtmp/no-docker-mark/a\") (literal \"$gmark/a\")) +(deny file-read-data (literal \"$gmark/a\")) (deny file-read* (regex #\"^/dev/ttys[0-9]+\$\")) (deny user-preference-write) (deny file-ioctl (ioctl-command #x80017472)) @@ -568,10 +573,46 @@ print("own-group", g.wait()) # replaced from inside, and no new one can be created. # On dfb6ee4: the profile allowed writes to /private/tmp/sh-thd*, so every # operation on that canary succeeded. - t21a=/private/tmp/sh-thd-flowcrm-test-$nonce - t21b=/private/var/tmp/sh-thd-flowcrm-test-$nonce + # Both directories are shared with other users: each canary gets a + # random name and is created exclusively, without following a link; it + # is read and removed only while it is still the file created here + # (same device and inode, a regular file of ours). + t21_file=' +import os, secrets, stat, sys +mode = sys.argv[1] +def same(path, dev, ino): + st = os.lstat(path) + return stat.S_ISREG(st.st_mode) and st.st_uid == os.getuid() and (st.st_dev, st.st_ino) == (int(dev), int(ino)) +if mode == "create": + path = os.path.join(sys.argv[2], "sh-thd-flowcrm-test-" + secrets.token_hex(12)) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + os.write(fd, b"original\n") + st = os.fstat(fd) + os.close(fd) + print(path, st.st_dev, st.st_ino) +elif mode == "read": + path, dev, ino = sys.argv[2:5] + try: + if not same(path, dev, ino): + print("replaced") + sys.exit(0) + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + print("same" if (os.fstat(fd).st_dev, os.fstat(fd).st_ino) == (int(dev), int(ino)) and os.read(fd, 64) == b"original\n" else "changed") + os.close(fd) + except FileNotFoundError: + print("missing") +elif mode == "remove": + path, dev, ino = sys.argv[2:5] + try: + if same(path, dev, ino): + os.unlink(path) + except FileNotFoundError: + pass +' + read -r t21a t21a_dev t21a_ino < <(python3 -I -c "$t21_file" create /private/tmp) || exit 2 + read -r t21b t21b_dev t21b_ino < <(python3 -I -c "$t21_file" create /private/var/tmp) || exit 2 + case "$t21a $t21b" in /private/tmp/sh-thd-flowcrm-test-*" "/private/var/tmp/sh-thd-flowcrm-test-*) ;; *) echo "HARNESS ERROR: T-21 canaries not created" >&2; exit 2 ;; esac w21=$(new_work) || exit 2 - for f in "$t21a" "$t21b"; do (umask 077 && printf 'original\n' >"$f") || exit 2; done t21=$("$nd" --work "$w21" python3 -I -c ' import errno, os, sys w = sys.argv[1] @@ -600,15 +641,19 @@ for path in sys.argv[2:]: ' "$w21" "$t21a" "$t21b" 2>/dev/null) r=$? t21_left="" + t21s=$(python3 -I -c "$t21_file" read "$t21a" "$t21a_dev" "$t21a_ino") + [ "$t21s" = same ] || t21_left="$t21_left $t21s:$t21a" + t21s=$(python3 -I -c "$t21_file" read "$t21b" "$t21b_dev" "$t21b_ino") + [ "$t21s" = same ] || t21_left="$t21_left $t21s:$t21b" for f in "$t21a" "$t21b"; do - [ "$(cat "$f" 2>/dev/null)" = original ] || t21_left="$t21_left changed:$f" - for x in "$f.moved" "$f.new"; do [ ! -e "$x" ] || t21_left="$t21_left $x"; done + for x in "$f.moved" "$f.new"; do [ ! -e "$x" ] && [ ! -L "$x" ] || t21_left="$t21_left $x"; done done printf '%s\n' "$t21" | grep '^t21 bad' | sed 's/^/INFO /' [ "$r" -eq 0 ] && [ "$(printf '%s\n' "$t21" | grep -c '^t21 ok')" -eq 18 ] && [ -z "$t21_left" ] result "T-21 bash 3.2 here-document files (/private/tmp and /private/var/tmp sh-thd-*) cannot be written, replaced or created from inside" $? \ "(rc $r, $(printf '%s\n' "$t21" | grep -c '^t21 ok') of 18 denied, changed or left:${t21_left:- none})" - rm -f "$t21a" "$t21b" "$t21a.moved" "$t21b.moved" "$t21a.new" "$t21b.new" + python3 -I -c "$t21_file" remove "$t21a" "$t21a_dev" "$t21a_ino" + python3 -I -c "$t21_file" remove "$t21b" "$t21b_dev" "$t21b_ino" # --- T-22 a file outside the run's directory cannot be changed through a # hard link, a symbolic link, a rename over it or an exchange (all on @@ -707,6 +752,9 @@ if how == "kill-self": if how == "wait": open(w + "/pids", "a").write(f"{os.getpid()}\n") time.sleep(60) +if how == "linger": + open(w + "/started", "w").write(f"{time.time():.3f}\n") + time.sleep(3) ' t24_left() { # run-dir tag: the recorded pids that still run as the case's processes local p @@ -827,6 +875,221 @@ ${t26_mark#"$ndtmp"/} t27 --orphans t27 --supervise + # --- T-30 to T-34 a run is reported done only when it is shown that none + # of its processes runs any more (B-1). no-docker-probe.py takes one copy + # of the process table at a time and needs a copy in which every process + # is shown not to be one of the run's, or was dead before that copy; + # otherwise the run fails with 125 and keeps its marker directory, and + # the next run refuses until that is shown. + # On e8ed7ed: one process table that happened to show no member ended + # the run as a success. + # + # T-30: a "hopper", each generation forking the next and ending, inside + # the sandbox: bounded (it ends by itself 18 s after it starts, or after + # 50000 generations) and recording a heartbeat in its run + # directory. The run must not be reported done while it hops; the next + # run must refuse while it hops; after it ended, a run starts again (and + # removes the old marker directory). + t30_hop=' +import os, sys, time +beat, deadline = sys.argv[1], float(sys.argv[2]) +g = 0 +while g < 50000 and time.time() < deadline: + g += 1 + if g % 100 == 0: + with open(beat, "a") as f: + f.write(f"{time.time():.3f} {g}\n") + if os.fork(): + os._exit(0) +' + t30_beats() { count_in . "$1"; } + w30=$(new_work) || exit 2 + t30_end=$(($(date +%s) + 18)) + t0=$(date +%s) + "$nd" --work "$w30" python3 -I -B -c "$t30_hop" "$w30/beat" "$t30_end" "t30-$nonce" >"$work/t30.out" 2>&1 + r=$? + took=$(($(date +%s) - t0)) + n0=$(t30_beats "$w30/beat") + sleep 1 + n1=$(t30_beats "$w30/beat") + t30_hopping=no + [ "$n1" -gt "$n0" ] && t30_hopping=yes + w30b=$(new_work) || exit 2 + "$nd" --work "$w30b" /usr/bin/touch "$w30b/ran" >"$work/t30b.out" 2>&1 + r2=$? + # Still hopping after that run returned? Then it must have refused. + n2=$(t30_beats "$w30/beat") + sleep 1 + t30_still=no + [ "$(t30_beats "$w30/beat")" -gt "$n2" ] && t30_still=yes + while [ "$(date +%s)" -le $((t30_end + 2)) ]; do sleep 1; done + w30c=$(new_work) || exit 2 + "$nd" --work "$w30c" /usr/bin/touch "$w30c/ran" >"$work/t30c.out" 2>&1 + r3=$? + # Not done while it hops: either the run failed (125, "could not be + # shown") or, if it ended anyway, the hopper had stopped. + { [ "$r" -eq 125 ] && grep -q 'could not be shown to have ended' "$work/t30.out"; } || + { [ "$r" -ne 125 ] && [ "$t30_hopping" = no ]; } + t30_ok1=$? + [ "$t30_ok1" -eq 0 ] && [ "$n0" -gt 0 ] && + { [ "$t30_still" = no ] || { [ "$r2" -eq 125 ] && [ ! -e "$w30b/ran" ]; }; } && + [ "$r3" -eq 0 ] && [ -e "$w30c/ran" ] + result "T-30 a process that keeps forking and ending is never reported as ended; new runs refuse while it runs, and start after it ended" $? \ + "(run rc $r after ${took}s, hopping afterwards $t30_hopping; next run rc $r2, hopping after it $t30_still; after its end rc $r3)" + + # T-31: two runs at the same time, each leaving a daemon: both are + # reported done, neither refuses the other, each ends its own daemon. + w31a=$(new_work) || exit 2 + w31b=$(new_work) || exit 2 + "$nd" --work "$w31a" python3 -I -B -c "$t24_leaver" "$w31a" "t31a-$nonce" linger >"$work/t31a.out" 2>&1 & + t31pa=$! + "$nd" --work "$w31b" python3 -I -B -c "$t24_leaver" "$w31b" "t31b-$nonce" linger >"$work/t31b.out" 2>&1 & + t31pb=$! + wait "$t31pa" + ra=$? + wait "$t31pb" + rb=$? + sleep 0.5 + la=$(t24_left "$w31a" "t31a-$nonce") + lb=$(t24_left "$w31b" "t31b-$nonce") + t24_clean "$w31a" "t31a-$nonce" + t24_clean "$w31b" "t31b-$nonce" + [ "$ra" -eq 0 ] && [ "$rb" -eq 0 ] && regular "$w31a/started" && regular "$w31b/started" && + [ "$(count_in . "$w31a/pids")" -eq 4 ] && [ "$(count_in . "$w31b/pids")" -eq 4 ] && [ -z "$la$lb" ] && + ! grep -q 'refused' "$work/t31a.out" "$work/t31b.out" + result "T-31 two runs at the same time each end their own processes and do not refuse each other" $? \ + "(rc $ra and $rb, survivors:${la}${lb:- none})" + + # T-32: unrelated processes that start and end while the run's + # processes are checked (here: this suite's own short-lived /usr/bin/true + # runs, outside) do not make the run fail: a process that ended before + # it could be checked is ended, not unknown, once a later copy of the + # table no longer has it. + w32=$(new_work) || exit 2 + ( i=0; while [ "$i" -lt 3000 ]; do /usr/bin/true; i=$((i + 1)); done ) & + t32churn=$! + "$nd" --work "$w32" python3 -I -B -c "$t24_leaver" "$w32" "t32-$nonce" ok >"$work/t32.out" 2>&1 + r=$? + # The churn must still have been running when the run ended. + t32_overlap=no + ps -o args= -p "$t32churn" 2>/dev/null | grep -q . && t32_overlap=yes + wait "$t32churn" + sleep 0.5 + left=$(t24_left "$w32" "t32-$nonce") + t24_clean "$w32" "t32-$nonce" + [ "$r" -eq 0 ] && [ "$t32_overlap" = yes ] && [ "$(count_in . "$w32/pids")" -eq 4 ] && [ -z "$left" ] + result "T-32 processes that start and end outside meanwhile do not make the ending fail" $? \ + "(rc $r, churn still running at the end $t32_overlap, survivors:${left:- none})" + + # T-33: if the ending cannot be shown (here: this case removes the run's + # marker file A while the command waits, then sends TERM), the run fails + # (125), the supervisor's own child does not survive, and the marker + # directory is kept. The case then ends the leftovers it recorded (by + # pid, while their arguments carry its tag), removes that directory and + # checks that a new run starts. + w33=$(new_work) || exit 2 + t33_before=$(cd "$ndtmp" && ls -d no-docker-mark.* 2>/dev/null) + python3 -I -c 'import os, signal, sys +signal.signal(signal.SIGINT, signal.SIG_DFL) +os.execv(sys.argv[1], sys.argv[1:])' "$nd" --work "$w33" python3 -I -B -c "$t24_leaver" "$w33" "t33-$nonce" wait >"$work/t33.out" 2>&1 & + ndpid=$! + for _ in $(seq 1 100); do [ "$(count_in . "$w33/pids")" -ge 5 ] && regular "$w33/mark" && break; sleep 0.1; done + t33_mark="" + regular "$w33/mark" && t33_mark=$(cat "$w33/mark") + case ${t33_mark#"$ndtmp"/} in no-docker-mark.*[!A-Za-z0-9_]* | "$t33_mark") t33_mark="" ;; no-docker-mark.?*) ;; *) t33_mark="" ;; esac + case " +$t33_before +" in *" +${t33_mark#"$ndtmp"/} +"*) t33_mark="" ;; esac + [ -n "$t33_mark" ] && [ -d "$t33_mark" ] && [ ! -L "$t33_mark" ] && [ -f "$t33_mark/a" ] && [ ! -L "$t33_mark/a" ] && + regular "$t33_mark/owner" && [ "$(cut -d' ' -f1 "$t33_mark/owner")" = "$ndpid" ] || t33_mark="" + t33_main=$(sed -n 5p "$w33/pids" 2>/dev/null | tr -cd '0-9') + r=none + if [ -n "$t33_mark" ] && ps -o args= -p "$ndpid" 2>/dev/null | grep -q -- no-docker; then + rm -f "$t33_mark/a" + kill -TERM "$ndpid" + wait "$ndpid" + r=$? + else + ps -o args= -p "$ndpid" 2>/dev/null | grep -q -- no-docker && kill -TERM "$ndpid" + wait "$ndpid" 2>/dev/null + fi + t33_child=gone + [ -n "$t33_main" ] && ps -o args= -p "$t33_main" 2>/dev/null | grep -q -- "t33-$nonce" && t33_child=alive + t33_kept=no + [ -n "$t33_mark" ] && [ -d "$t33_mark" ] && t33_kept=yes + t24_clean "$w33" "t33-$nonce" + [ -n "$t33_mark" ] && rm -rf "$t33_mark" + w33b=$(new_work) || exit 2 + "$nd" --work "$w33b" /usr/bin/touch "$w33b/ran" >/dev/null 2>&1 + r2=$? + [ "$r" = 125 ] && grep -q 'could not be shown to have ended' "$work/t33.out" && [ "$t33_child" = gone ] && + [ "$t33_kept" = yes ] && [ "$r2" -eq 0 ] && [ -e "$w33b/ran" ] + result "T-33 an ending that cannot be shown fails the run (125), ends the supervisor's own child and keeps the marker" $? \ + "(rc $r, command $t33_child, marker kept $t33_kept, next run rc $r2)" + + # T-34: TERM to no-docker.sh at several moments of the start, while the + # command starts processes: each run ends within 15 s, as interrupted + # (143) or as not shown (125); none of the processes it recorded + # survives once the run ended or was ended through its marker. + t34_spawn=' +import os, sys, time +w, tag = sys.argv[1], sys.argv[2] +open(w + "/mark", "w").write(os.environ.get("NO_DOCKER_MARK", "")) +SLEEP = [sys.executable, "-I", "-B", "-c", "import os, sys, time; open(sys.argv[1], \"a\").write(f\"{os.getpid()}\\n\"); time.sleep(30)", w + "/pids", tag] +for _ in range(20): + if os.fork() == 0: + os.setsid() + os.execv(SLEEP[0], SLEEP) + time.sleep(0.05) +time.sleep(30) +' + t34_bad="" + for t34_delay in 0 0.1 0.3 0.8; do + w34=$(new_work) || exit 2 + t34_before=$(cd "$ndtmp" && ls -d no-docker-mark.* 2>/dev/null) + python3 -I -c 'import os, signal, sys +signal.signal(signal.SIGINT, signal.SIG_DFL) +os.execv(sys.argv[1], sys.argv[1:])' "$nd" --work "$w34" python3 -I -B -c "$t34_spawn" "$w34" "t34-$nonce" >"$work/t34.out" 2>&1 & + ndpid=$! + sleep "$t34_delay" + t0=$(date +%s) + ps -o args= -p "$ndpid" 2>/dev/null | grep -q -- no-docker && kill -TERM "$ndpid" + wait "$ndpid" + r=$? + took=$(($(date +%s) - t0)) + sleep 0.5 + left=$(t24_left "$w34" "t34-$nonce") + if [ "$r" -eq 125 ]; then + # Not shown: the kept marker directory of this run ends what is left. + # It is the one the command recorded, if that is a directory that did + # not exist before this iteration and whose owner record names this + # run's no-docker.sh (now gone); --end also refuses a directory whose + # owner still runs. + t34_mark="" + regular "$w34/mark" && t34_mark=$(cat "$w34/mark") + case ${t34_mark#"$ndtmp"/} in no-docker-mark.*[!A-Za-z0-9_]* | "$t34_mark") t34_mark="" ;; no-docker-mark.?*) ;; *) t34_mark="" ;; esac + case " +$t34_before +" in *" +${t34_mark#"$ndtmp"/} +"*) t34_mark="" ;; esac + if [ -n "$t34_mark" ] && [ -d "$t34_mark" ] && [ ! -L "$t34_mark" ] && regular "$t34_mark/owner" && + [ "$(cut -d' ' -f1 "$t34_mark/owner")" = "$ndpid" ]; then + python3 -I "$root/scripts/ci/no-docker-probe.py" --end "$t34_mark" >/dev/null 2>&1 + fi + sleep 0.5 + left=$(t24_left "$w34" "t34-$nonce") + fi + t24_clean "$w34" "t34-$nonce" + case $r in 143 | 125) ;; *) t34_bad="$t34_bad delay$t34_delay:rc$r" ;; esac + [ "$took" -le 15 ] || t34_bad="$t34_bad delay$t34_delay:${took}s" + [ -z "$left" ] || t34_bad="$t34_bad delay$t34_delay:survivors$left" + done + [ -z "$t34_bad" ] + result "T-34 TERM while the command starts processes: every run ends in time and none of its processes survives" $? "(${t34_bad:- all ok})" + # --- T-28 no debugger access to a process outside (A-M2). The canaries # are copies of /bin/sleep in this suite's directory, signed ad hoc with # get-task-allow, so that a debugger of the same user may attach to them; @@ -904,7 +1167,7 @@ print(" ".join(out)) [ "$t29" = "outside:read inside:denied 1" ] result "T-29 another terminal's device cannot be read from inside (control: it can from outside)" $? "($t29)" else - echo "INFO T-06, T-08 to T-15, T-21, T-22, T-24 to T-29 skipped: macOS sandbox profile only" + echo "INFO T-06, T-08 to T-15, T-21, T-22, T-24 to T-34 skipped: macOS sandbox profile only" fi # --- T-07 loopback ports listening before the isolation From 980d6a5f0f5c9808fae4583d21c54945334d44d9 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 18:51:10 +0100 Subject: [PATCH 28/77] docs(dev-environment): describe how a macOS run is shown to have ended The supervisor's proof over copies of the process list, the 125 when it cannot be given (measured for a process that keeps forking and ending, which is contained but not ended), the owner record of each run and the new regression cases. --- docs/dev-environment.md | 116 +++++++++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 43 deletions(-) diff --git a/docs/dev-environment.md b/docs/dev-environment.md index ea077f4..154a512 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -530,42 +530,57 @@ directory. The code that runs inside (`bin/devenv`, `tests/devenv/unit.sh`, the guard suites and the scripts they run) therefore uses none; T-20 checks this statically. -When the command ends, every process it left in the sandbox is ended: a +When the command ends, the processes it left in the sandbox are ended: a plain background job, a daemon after `setsid` and a double fork, a `posix_spawn` child in a session of its own, a process group of its own. `no-docker.sh` hands its process to a supervisor in `scripts/ci/no-docker-probe.py`, which starts `sandbox-exec`, waits for it and then ends what is left. It finds those processes by a marker: the -profile denies reading file `a` of a marker directory of the run (and of -one common to every run), in `$TMPDIR` outside the run's directory, and -leaves file `b` next to it alone. From outside, `sandbox_check()` asks how a -process's sandbox would decide; a process is in the run's sandbox exactly -when it is denied `a` and allowed `b`. Measured: an unsandboxed process is -allowed both, other sandboxes decide both the same way, and the files must -exist, or every process is "denied". A process cannot leave its sandbox, -and every process it starts inherits it, however it detaches. The -supervisor sends SIGKILL to each member as soon as it finds one, after -reading its start time and membership again, and lists the processes again -until there is none. A killed process cannot start another one, and -nothing a member does can undo SIGKILL. The supervisor needs no new process -to do this, so a command that uses up the processes it may start cannot -stop it. It prints how many processes it ended. If members are still -there after 10 seconds (one of another user, or members that start new -processes faster than they are ended), the run fails with status 125. The -run's marker directory is then kept, the message names the command that -ends them (`no-docker-probe.py --end `), and until then no -new run starts. SIGINT, SIGTERM, SIGHUP or SIGQUIT delivered to `no-docker.sh` -end the command and everything it left at once, and the status is 128 plus -the signal number; a signal the caller ignores stays ignored. The probe -checks from inside that the sandbox carries both markers. +profile denies reading file `a` of the run's marker directory, in `$TMPDIR` +outside the run's directory, and leaves file `b` next to it alone. From +outside, `sandbox_check()` asks how a process's sandbox would decide; a +process is in the run's sandbox exactly when it is sandboxed, denied `a` and +allowed `b`. Measured: an unsandboxed process is allowed both, and the +files must exist, or every process is "denied". A process cannot leave its +sandbox, and every process it starts inherits it, however it detaches. + +The run is reported done only when it is shown that none of its processes +runs any more. The supervisor takes one copy of the kernel's process list +at a time (the kernel copies it under one lock, so it has every process of +that instant, zombies included) and checks each process in it, by its +unique id before and after the check: one of the run's is sent SIGKILL at +once. A copy shows the run has ended when every process in it is shown not +to be one of the run's, or was already dead before that copy was taken. +Every process started later descends from one in the copy (the kernel lists +a child before it can run, and its parent until it is reaped), so none of +them can be one of the run's either. A process that ends before it can be +checked leaves that copy open, as it may have started one of the run's +first; the supervisor then takes the next copy. Measured: a run without +leftovers is shown done in about 0.07 seconds, a run whose daemon must be +ended in about 0.4 seconds. + +If that cannot be shown within 10 seconds, the run fails with status 125; +it is never reported done. Measured: a process that forks a new one and +ends, over and over (thousands of times per second), makes every copy +open: the run fails, although that process is not found and keeps running, +still in the sandbox with every limit above. The run's marker directory is +then kept, and no new run starts until it is shown that none of its +processes runs any more (then that directory is removed). The supervisor +needs no new process for any of this, so a command that uses up the +processes it may start cannot stop it. SIGINT, SIGTERM, SIGHUP or SIGQUIT +delivered to `no-docker.sh` end the command and everything it left in the +same way, and the status is 128 plus the signal number; a signal the +caller ignores stays ignored. The probe checks from inside that the +sandbox carries the run's marker. SIGKILL of `no-docker.sh` (or a crash of the supervisor) cannot be handled: what the command left keeps running, still inside the sandbox with every -limit above. Before each run, `no-docker.sh` therefore looks for processes -of the common marker that run on their own (their supervisor is gone) and -refuses to start while there is one, naming them. A concurrent run's own -processes are under its supervisor and do not count, but a daemon it left -does until it ends. +limit above. The marker directory records the run's owner (the process +that becomes its supervisor). Before each run, `no-docker.sh` looks at the +marker directories whose owner is gone; it removes one once it is shown +that none of its processes runs, and otherwise refuses to start, naming +them. It never ends anything there. Runs in progress elsewhere (their +owner lives) are left alone. Setuid programs cannot run in the sandbox, so `ps` is an ad-hoc signed copy and `sudo` is unavailable. The U-16 check that another local user cannot @@ -701,19 +716,21 @@ them. #### Accepted residual risks -- **SIGKILL of `no-docker.sh` (macOS), or a failed ending.** Its - supervisor cannot end the command or what it left. They keep running in - the sandbox, with every limit above, until someone ends them - (`no-docker-probe.py --end `, or by pid); meanwhile no - new isolated run starts. Until then the channels the sandbox leaves open - (their own terminal, loopback ports, the pasteboard, reading the user's - files and process information) stay available to them, without a time - limit. Processes left by runs of a version without the markers (before - the supervisor) are not recognised; end them by pid after updating. +- **SIGKILL of `no-docker.sh` (macOS), or an ending that cannot be shown.** + The command and what it left keep running in the sandbox, with every + limit above, until someone ends them (`no-docker-probe.py --end `, or by pid); meanwhile no new isolated run starts. A process + that keeps forking and ending is not found at all; it runs until it stops + by itself, the user ends it or logs out. Until then the channels the + sandbox leaves open (loopback ports, the pasteboard, reading the user's + files and process information) stay available to it, without a time + limit. Processes left by versions without the owner record are not + recognised; end them by pid after updating. - **Process identity (macOS).** A process is signalled only after its - start time and membership were read again, microseconds before. A pid + unique id and membership were read again, microseconds before. A pid reused within that window by a process outside could still be ended: - an availability risk, not a way out. + an availability risk, not a way out. Reusing a pid needs the kernel's + counter to wrap (pids up to 99999). - **Late listeners.** A loopback port that starts listening after the run started (on macOS) is reachable, as are the ports the command opens itself. This includes a VM's port forward that starts later. The probe @@ -827,7 +844,7 @@ in the test's directories or send signal 0 or TERM to the test's own - **T-01 and T-02:** `--check`, and a claimed but unproven isolation. - **T-07:** loopback ports that were already listening are unreachable, while ports opened inside are reachable. -- **T-06, T-08 to T-15, T-21, T-22 and T-24 to T-29, macOS only:** +- **T-06, T-08 to T-15, T-21, T-22 and T-24 to T-34, macOS only:** - **T-06 and T-11:** the write limits, including that a bash 3.2 here-document fails; - **T-08:** the complete port list; @@ -838,8 +855,10 @@ in the test's directories or send signal 0 or TERM to the test's own - **T-14:** integrity of the helpers; - **T-15:** `--work`, accepted and refused cases; - **T-21:** canary files named like bash 3.2 here-document files in - `/tmp` and `/private/var/tmp` cannot be opened for writing, truncated, - removed, renamed, replaced or chmodded, and none can be created; + `/tmp` and `/private/var/tmp` (random names, created exclusively, read + and removed only while they are the files created) cannot be opened for + writing, truncated, removed, renamed, replaced or chmodded, and none can + be created; - **T-22:** a file outside the run's directory cannot be changed through a hard link, a symbolic link, a rename over it or an exchange; - **T-24:** four processes the command leaves (a background child, a @@ -849,11 +868,22 @@ in the test's directories or send signal 0 or TERM to the test's own - **T-25:** SIGINT, SIGTERM and SIGHUP to `no-docker.sh` end the command and those four processes at once (status 128 plus the signal); - **T-26:** after SIGKILL of `no-docker.sh`, its leftovers keep a new run - from starting (125) until they are ended through the run's marker; + from starting (125) until they are ended through the run's marker + directory, whose owner is gone; - **T-27:** when the probe cannot make the markers, look for leftovers or supervise, the command is not run; - **T-28:** a debugger attach from inside to a test canary outside that allows debuggers fails, while it works from outside; + - **T-30:** a process that keeps forking and ending is never reported as + ended; new runs refuse while it runs and start after it ended; + - **T-31:** two runs at the same time each end their own processes and + do not refuse each other; + - **T-32:** processes that start and end outside meanwhile do not make + the ending fail; + - **T-33:** an ending that cannot be shown fails the run (125), ends the + supervisor's own child and keeps the marker directory; + - **T-34:** SIGTERM at several moments while the command starts processes: + every run ends in time and none of its processes survives; - **T-29:** another terminal's device cannot be read from inside, while it can from outside. - **T-16 to T-18, Linux only:** From 8e048abbfcad4e8590d57be77a6ec0f42426c349 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 19:12:07 +0100 Subject: [PATCH 29/77] fix(test): count zombies and a gone owner's children correctly when ending a macOS run A zombie is found only with an argument of 1 to proc_pidinfo; read as vanished, one that lives long kept every copy of the process table open and failed every run with 125. It now counts as dead from the next copy on. When a run whose owner is gone is checked (before a run, or --end), a process that owner started and that is not yet a member keeps the copy open, as it may be the run's sandbox-exec before it applied the profile. Waiting for the supervisor's own child after killing it is bounded. --- scripts/ci/no-docker-probe.py | 70 +++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/scripts/ci/no-docker-probe.py b/scripts/ci/no-docker-probe.py index 406a071..f4730d7 100644 --- a/scripts/ci/no-docker-probe.py +++ b/scripts/ci/no-docker-probe.py @@ -364,9 +364,9 @@ def snapshot(self): return [buf[i] for i in range(got) if buf[i] > 0 and buf[i] != os.getpid()] raise OSError("the process list did not fit") - def _info(self, pid, flavor, struct): + def _info(self, pid, flavor, struct, arg=0): ctypes.set_errno(0) - n = self.proc.proc_pidinfo(pid, flavor, 0, ctypes.byref(struct), ctypes.sizeof(struct)) + n = self.proc.proc_pidinfo(pid, flavor, arg, ctypes.byref(struct), ctypes.sizeof(struct)) if n == ctypes.sizeof(struct): return True if ctypes.get_errno() in (errno.ESRCH, 0): @@ -374,10 +374,21 @@ def _info(self, pid, flavor, struct): raise OSError(f"cannot read process {pid} ({os.strerror(ctypes.get_errno())})") def identity(self, pid): - """The kernel's unique id of the process, or None if it is gone.""" + """The kernel's unique id of the process, or None if it is gone (or + a zombie).""" u = ProcUniqIdentifierInfo() return u.uniqueid if self._info(pid, self.PROC_PIDUNIQIDENTIFIERINFO, u) else None + def parent_identity(self, pid): + """The unique id of the process's parent, or None if it is gone.""" + u = ProcUniqIdentifierInfo() + return u.puniqueid if self._info(pid, self.PROC_PIDUNIQIDENTIFIERINFO, u) else None + + def zombie(self, pid): + """True if pid is a zombie (only an argument of 1 finds those).""" + s = ProcShortInfo() + return self._info(pid, self.PROC_PIDT_SHORTBSDINFO, s, arg=1) and s.status == SZOMB + class Sandboxes(ProcTable): """The process table and sandbox_check() for the marker of one run. @@ -425,6 +436,8 @@ def classify(self, pid): between two identity reads, so the answer belongs to one process.""" first = self.identity(pid) if first is None: + if self.zombie(pid): + return "zombie", None raise Gone() s = ProcShortInfo() if not self._info(pid, self.PROC_PIDT_SHORTBSDINFO, s): @@ -441,7 +454,7 @@ def classify(self, pid): return ("member" if sandboxed == 1 and a == 1 and b == 0 else "other"), first -def sweep(sbx, seconds, kill): +def sweep(sbx, seconds, kill, owner=None): """Show that no process of the sandbox runs any more, killing each one found if kill (SIGKILL, after its identity and the markers were read again). Returns (ended, members) once a round proves it: in one @@ -453,7 +466,13 @@ def sweep(sbx, seconds, kill): before it could be told apart, or became a zombie only now, leaves the round open: it may have started a member before it died. Raises OSError when no round proves it within seconds (for example a process - that keeps starting new ones and ending); the caller fails closed.""" + that keeps starting new ones and ending); the caller fails closed. + owner (the unique id of a run's owner that is gone): a process the + owner started that is not a member (yet) keeps every round open, as it + may be the run's sandbox-exec before it applied the profile, the one + process that becomes a member after it was checked. Relies on pids not + being reused within one round (the kernel's counter would have to wrap + at 99999).""" deadline = time.monotonic() + seconds others = {} # pid: unique id of a process that is not a member dead = set() # pids seen dead before the current snapshot @@ -477,6 +496,9 @@ def sweep(sbx, seconds, kill): proven = False continue if kind == "other": + if owner is not None and sbx.parent_identity(pid) == owner: + proven = False + continue others[pid] = uid elif kind == "zombie": dead_now.add(pid) @@ -506,18 +528,25 @@ def sweep(sbx, seconds, kill): time.sleep(0.005) -def owner_alive(table, directory): - """True if the owner the marker directory records still runs (same pid - and unique id), False if it is gone; raises OSError if the record is - unreadable.""" +def owner_record(directory): + """(pid, unique id) the marker directory records as its owner; raises + OSError if the record is unreadable.""" rec = os.path.join(directory, "owner") if not regular_file(rec) or os.lstat(rec).st_size > 64: raise OSError(f"{rec} is not a regular file of ours") with open(rec) as f: fields = f.read().split() - if len(fields) != 2 or not fields[0].isdigit(): + if len(fields) != 2 or not fields[0].isdigit() or not fields[1].isdigit(): raise OSError(f"{rec} is not an owner record") - return str(table.identity(int(fields[0]))) == fields[1] + return int(fields[0]), int(fields[1]) + + +def owner_alive(table, directory): + """True if the owner the marker directory records still runs (same pid + and unique id), False if it is gone; raises OSError if the record is + unreadable.""" + pid, uid = owner_record(directory) + return table.identity(pid) == uid def darwin_markers(tmp, owner): @@ -578,7 +607,11 @@ def darwin_orphans(tmp, seconds=5.0): if not os.path.lexists(d): continue # removed meanwhile by its own run try: - _, members = sweep(Sandboxes(d), seconds, kill=False) + try: + gone_owner = owner_record(d)[1] + except FileNotFoundError: + gone_owner = None + _, members = sweep(Sandboxes(d), seconds, kill=False, owner=gone_owner) except OSError as e: # Among others: a marker file is missing, so its processes # cannot be found any more; the directory stays until someone @@ -674,14 +707,17 @@ def on_signal(signum, frame): # still its own (also when sandbox-exec has not applied the # profile yet). child.kill() - status = child.wait() + status = child.wait(timeout=10) ended = darwin_end(sbx) except BaseException as e: # noqa: BLE001 - every failure fails closed failure = str(e) or type(e).__name__ finally: if child is not None and child.poll() is None: child.kill() - child.wait() + try: + child.wait(timeout=10) + except subprocess.TimeoutExpired: + failure = failure or f"the isolation tool (pid {child.pid}) does not end" started = os.path.exists(started_file) if not failure: shutil.rmtree(markdir, ignore_errors=True) @@ -1109,7 +1145,11 @@ def main(): alive = False if alive: raise OSError(f"the run of {sys.argv[2]} is still in progress (its owner runs); not ending it") - n = darwin_end(sbx) + try: + gone_owner = owner_record(sys.argv[2])[1] + except FileNotFoundError: + gone_owner = None + n = sweep(sbx, 10.0, kill=True, owner=gone_owner)[0] shutil.rmtree(sys.argv[2], ignore_errors=True) print(f"no-docker: ended {n} processes; none is left", file=sys.stderr) return 0 From 4fca2a287b54b66efee7125748d2b2ac61d2c703 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 19:12:07 +0100 Subject: [PATCH 30/77] test(ci): send TERM while processes are being started and cover a zombie outside T-34 now waits until the command recorded its first process, sends TERM 0, 0.15 and 0.4 s later, requires processes to have been recorded and accepts 125 only with its message. T-35 keeps a zombie of the suite's own outside while a run with leftovers must end with status 0. T-30 and T-34 print what they observed. --- scripts/ci/tests/no-docker.test.sh | 60 +++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/scripts/ci/tests/no-docker.test.sh b/scripts/ci/tests/no-docker.test.sh index 1c8021f..241fb99 100755 --- a/scripts/ci/tests/no-docker.test.sh +++ b/scripts/ci/tests/no-docker.test.sh @@ -48,7 +48,7 @@ # marker file of a run of their own (T-33), or sleep. # This suite itself runs no docker command. # -# T-06, T-08 to T-15, T-21, T-22 and T-24 to T-34 are macOS only, T-16 to +# T-06, T-08 to T-15, T-21, T-22 and T-24 to T-35 are macOS only, T-16 to # T-18 Linux only (each prints INFO on the other platform); T-19, T-20 and # T-23 are static and run on both. Each case says how it fails against the # no-docker.sh of 38b5ebd, dfb6ee4 or e8ed7ed, which these cases were @@ -934,6 +934,9 @@ while g < 50000 and time.time() < deadline: [ "$t30_ok1" -eq 0 ] && [ "$n0" -gt 0 ] && { [ "$t30_still" = no ] || { [ "$r2" -eq 125 ] && [ ! -e "$w30b/ran" ]; }; } && [ "$r3" -eq 0 ] && [ -e "$w30c/ran" ] + t30_ok=$? + echo "INFO T-30 run rc $r after ${took}s, hopping afterwards $t30_hopping; next run rc $r2, hopping after it $t30_still; after its end rc $r3" + [ "$t30_ok" -eq 0 ] result "T-30 a process that keeps forking and ending is never reported as ended; new runs refuse while it runs, and start after it ended" $? \ "(run rc $r after ${took}s, hopping afterwards $t30_hopping; next run rc $r2, hopping after it $t30_still; after its end rc $r3)" @@ -1029,10 +1032,40 @@ ${t33_mark#"$ndtmp"/} result "T-33 an ending that cannot be shown fails the run (125), ends the supervisor's own child and keeps the marker" $? \ "(rc $r, command $t33_child, marker kept $t33_kept, next run rc $r2)" - # T-34: TERM to no-docker.sh at several moments of the start, while the - # command starts processes: each run ends within 15 s, as interrupted - # (143) or as not shown (125); none of the processes it recorded - # survives once the run ended or was ended through its marker. + # T-35: a zombie outside (a child of this suite that has ended and is + # not reaped for 15 s) does not make the ending unprovable: it is found + # as a zombie and counts as dead from the next copy of the table on. + # On 980d6a5 before the fix: zombies were not found (they read as + # vanished), so every copy stayed open and the run failed with 125. + python3 -I -c 'import subprocess, sys, time +p = subprocess.Popen(["/usr/bin/true"]) +open(sys.argv[1], "w").write(str(p.pid)) +time.sleep(15) +p.wait()' "$work/t35.zombie" & + t35holder=$! + for _ in $(seq 1 50); do regular "$work/t35.zombie" && break; sleep 0.1; done + sleep 0.5 + t35z=$(tr -cd '0-9' <"$work/t35.zombie" 2>/dev/null) + t35_state=$(ps -o stat= -p "${t35z:-0}" 2>/dev/null | tr -d ' ') + w35=$(new_work) || exit 2 + "$nd" --work "$w35" python3 -I -B -c "$t24_leaver" "$w35" "t35-$nonce" ok >"$work/t35.out" 2>&1 + r=$? + sleep 0.5 + left=$(t24_left "$w35" "t35-$nonce") + t24_clean "$w35" "t35-$nonce" + wait "$t35holder" + echo "INFO T-35 zombie ${t35z:-?} state ${t35_state:-gone} during the run; rc $r; survivors:${left:- none}" + case $t35_state in Z*) t35_zombie=yes ;; *) t35_zombie=no ;; esac + [ "$t35_zombie" = yes ] && [ "$r" -eq 0 ] && [ -z "$left" ] + result "T-35 a zombie outside does not make the ending of a run unprovable" $? "(zombie $t35_zombie, rc $r)" + + # T-34: TERM to no-docker.sh while the command is starting processes + # (at 0, 0.15 and 0.4 s after it recorded its first one; it starts one + # every 50 ms for 1 s): processes must have been recorded when TERM is + # sent; each run ends within 15 s, interrupted (143) or, if the ending + # could not be shown, 125 with that message; none of the processes it + # recorded survives once the run ended or was ended through its own + # marker directory. t34_spawn=' import os, sys, time w, tag = sys.argv[1], sys.argv[2] @@ -1046,19 +1079,22 @@ for _ in range(20): time.sleep(30) ' t34_bad="" - for t34_delay in 0 0.1 0.3 0.8; do + for t34_delay in 0 0.15 0.4; do w34=$(new_work) || exit 2 t34_before=$(cd "$ndtmp" && ls -d no-docker-mark.* 2>/dev/null) python3 -I -c 'import os, signal, sys signal.signal(signal.SIGINT, signal.SIG_DFL) os.execv(sys.argv[1], sys.argv[1:])' "$nd" --work "$w34" python3 -I -B -c "$t34_spawn" "$w34" "t34-$nonce" >"$work/t34.out" 2>&1 & ndpid=$! + for _ in $(seq 1 200); do [ "$(count_in . "$w34/pids")" -ge 1 ] && break; sleep 0.05; done sleep "$t34_delay" + t34_n=$(count_in . "$w34/pids") t0=$(date +%s) ps -o args= -p "$ndpid" 2>/dev/null | grep -q -- no-docker && kill -TERM "$ndpid" wait "$ndpid" r=$? took=$(($(date +%s) - t0)) + t34_all=$(count_in . "$w34/pids") sleep 0.5 left=$(t24_left "$w34" "t34-$nonce") if [ "$r" -eq 125 ]; then @@ -1083,7 +1119,13 @@ ${t34_mark#"$ndtmp"/} left=$(t24_left "$w34" "t34-$nonce") fi t24_clean "$w34" "t34-$nonce" - case $r in 143 | 125) ;; *) t34_bad="$t34_bad delay$t34_delay:rc$r" ;; esac + echo "INFO T-34 after ${t34_delay}s: $t34_n recorded at TERM, $t34_all in all, rc $r, ${took}s, survivors:${left:- none}" + [ "$t34_n" -ge 1 ] || t34_bad="$t34_bad delay$t34_delay:nothing-recorded-at-TERM" + case $r in + 143) ;; + 125) grep -q 'could not be shown to have ended' "$work/t34.out" || t34_bad="$t34_bad delay$t34_delay:125-without-message" ;; + *) t34_bad="$t34_bad delay$t34_delay:rc$r" ;; + esac [ "$took" -le 15 ] || t34_bad="$t34_bad delay$t34_delay:${took}s" [ -z "$left" ] || t34_bad="$t34_bad delay$t34_delay:survivors$left" done @@ -1167,7 +1209,7 @@ print(" ".join(out)) [ "$t29" = "outside:read inside:denied 1" ] result "T-29 another terminal's device cannot be read from inside (control: it can from outside)" $? "($t29)" else - echo "INFO T-06, T-08 to T-15, T-21, T-22, T-24 to T-34 skipped: macOS sandbox profile only" + echo "INFO T-06, T-08 to T-15, T-21, T-22, T-24 to T-35 skipped: macOS sandbox profile only" fi # --- T-07 loopback ports listening before the isolation @@ -1466,7 +1508,7 @@ check_safe() { # name [ "$STARTED" -eq 1 ] result "$1: the harness was started" $? if [ "$os" = Darwin ] && [ "$t0304" = bare ]; then - # The harness nests its own sandbox inside the guard and runs the mutant. + # The harness runs on the host and runs the mutant in its own sandbox. [ "$RC" -eq 97 ] && [ "$cases" -ge 20 ] result "$1: the mutant ran to its end inside the harness's isolation" $? "(rc $RC, $cases cases)" else From 85cfb37c30b437a1a822097198366e99c5a64c95 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 19:12:07 +0100 Subject: [PATCH 31/77] docs(dev-environment): name the exceptions the proof of a macOS run's end relies on The run's own sandbox-exec before it applies the profile, zombies as dead from the next copy on, pids not reused within one copy, and how marker directories of earlier versions are handled. --- docs/dev-environment.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/dev-environment.md b/docs/dev-environment.md index 154a512..f77f3d3 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -550,10 +550,16 @@ at a time (the kernel copies it under one lock, so it has every process of that instant, zombies included) and checks each process in it, by its unique id before and after the check: one of the run's is sent SIGKILL at once. A copy shows the run has ended when every process in it is shown not -to be one of the run's, or was already dead before that copy was taken. -Every process started later descends from one in the copy (the kernel lists -a child before it can run, and its parent until it is reaped), so none of -them can be one of the run's either. A process that ends before it can be +to be one of the run's, or was already dead (a zombie) before that copy was +taken. Every process started later descends from one in the copy (the +kernel lists a child before it can run, and its parent until it is +reaped), so none of them can be one of the run's either. The one process +that becomes one of the run's after it could be checked is the run's own +`sandbox-exec` before it applies the profile: the supervisor waits for it +first, and when a run is checked whose owner is gone, a process that owner +started keeps every copy open. The argument relies on a pid not being +reused while one copy is checked (milliseconds; the kernel's counter wraps +at 99999). A process that ends before it can be checked leaves that copy open, as it may have started one of the run's first; the supervisor then takes the next copy. Measured: a run without leftovers is shown done in about 0.07 seconds, a run whose daemon must be @@ -724,8 +730,10 @@ them. by itself, the user ends it or logs out. Until then the channels the sandbox leaves open (loopback ports, the pasteboard, reading the user's files and process information) stay available to it, without a time - limit. Processes left by versions without the owner record are not - recognised; end them by pid after updating. + limit. A marker directory of an earlier version (without the owner + record) is checked like one whose owner is gone once it is a minute old; + processes of such a version whose directory was already removed are not + recognised: end them by pid after updating. - **Process identity (macOS).** A process is signalled only after its unique id and membership were read again, microseconds before. A pid reused within that window by a process outside could still be ended: From f49e7e75ebcc369c1545ea5c96310c5c9c096e53 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 23:32:56 +0100 Subject: [PATCH 32/77] fix(test): prove the Linux pid namespace from the initial one and list live sockets from the kernel The probe required two NSpid entries, which a /proc of the run's own pid namespace never shows, so every Linux isolation refused (run 41126389201). The pid namespace is now shown below the caller's by the caller's being the initial one (its fixed inode number) and the run's not; NSpid must be the run's pid alone, and /proc must hide no process. The socket list no longer walks every file system on every run (about 140 s): as root, the kernel's socket diagnostics of every network namespace give every live pathname socket, used only when every socket held by a process is accounted for and every bound file is found again with one link on a local file system. Otherwise the unchanged walk runs. The checks are evaluated together and gate the writes and connects; an empty socket list and a system primary group fail. --- scripts/ci/no-docker-probe.py | 856 ++++++++++++++++++++++++++++++---- scripts/ci/no-docker.sh | 43 +- 2 files changed, 793 insertions(+), 106 deletions(-) diff --git a/scripts/ci/no-docker-probe.py b/scripts/ci/no-docker-probe.py index f4730d7..36f31a9 100644 --- a/scripts/ci/no-docker-probe.py +++ b/scripts/ci/no-docker-probe.py @@ -14,18 +14,22 @@ /private/var/tmp, while $TMPDIR (inside the run's directory) is writable; Linux: not uid 0, no capability in any set, no_new_privs, no - supplementary group; network, mount, pid and IPC namespaces that - differ from the caller's (NO_DOCKER_HOST_NS); a pid namespace below - the caller's whose pid 1 is the root shell of no-docker.sh and whose - other processes are all ours; a session of our own without a - controlling terminal; only the loopback interface; /tmp, /var/tmp, + supplementary group, a primary group that is not a system group; + network, mount, pid and IPC namespaces that differ from the caller's + (NO_DOCKER_HOST_NS); a pid namespace below the caller's (the caller's + is the initial one, ours is not), with a /proc of its own (NSpid is + our pid alone, /proc/self is our pid) that hides no process, whose + pid 1 is the root shell of no-docker.sh and whose other processes + are all ours; a session of our own without a controlling terminal; + no standard descriptor that is a Unix socket; only the loopback + interface; /tmp, /var/tmp, /dev/shm, /run and /dev/mqueue are new file systems (NO_DOCKER_HOST_DEVS) and /sys/fs/cgroup is masked; every other mount reachable by path is read-only; writes to $HOME, $GITHUB_WORKSPACE and $RUNNER_TEMP (NO_DOCKER_PROTECTED) fail; no cgroup file is writable; no socket in the private directories; and no Unix socket that no-docker.sh listed outside (NO_DOCKER_SOCKETS) - accepts a connection; + accepts a connection, of a list that is not empty; - and no container engine answers on any endpoint we can find: the usual socket paths of Docker Engine, Docker Desktop, rootless Docker, Podman, Colima, OrbStack and Rancher Desktop; every endpoint of the @@ -41,10 +45,16 @@ Modes used by no-docker.sh outside the isolation: --listening print the listening TCP port numbers (macOS), one per line; - --sockets print, NUL-separated, every Unix socket path on the local - file systems and every bound pathname socket of this - network namespace (Linux); fails when its time budget runs - out; + --sockets-fast + as root: print, NUL-separated, every path to every live + bound pathname Unix socket of every network namespace, from + the kernel's socket lists (Linux); exit 3, with the paths + found so far, unless the list is shown complete within 10 s; + --sockets [BUDGET] + the walk: print, NUL-separated, every Unix socket path on + the local file systems and every bound pathname socket of + this network namespace (Linux); fails when its time budget + (180 s) runs out; --seal WORK as root in the new mount namespace: make every mount reachable by path read-only, except the private file systems and WORK, and verify it (Linux); @@ -67,13 +77,16 @@ import ctypes import errno import glob +import grp import json import os import re +import select import shutil import signal import socket import stat +import struct import subprocess import sys import tempfile @@ -750,6 +763,13 @@ def on_signal(signum, frame): ("/dev/mqueue", "mqueue")) HOST_COMMS = {"systemd", "init", "Runner.Worker", "Runner.Listener", "dockerd", "containerd", "sshd", "containerd-shim", "rootlesskit", "podman", "dbus-daemon"} +# The initial pid namespace, from which every other one descends: its inode +# number is fixed (PROC_PID_INIT_INO); every other namespace gets a number +# of 0xF0000000 or above. +INIT_PID_NS = "pid:[4026531836]" +# Debian and Ubuntu give system groups (docker, disk, adm, shadow, +# systemd-journal, ...) gids up to 999. +SYS_GID_MAX = 999 def unescape(s): @@ -763,7 +783,8 @@ def mounts(): fields = line.split() sep = fields.index("-") entries.append({"id": int(fields[0]), "point": unescape(fields[4]), - "opts": fields[5].split(","), "fstype": fields[sep + 1]}) + "opts": fields[5].split(","), "fstype": fields[sep + 1], + "super": fields[sep + 3].split(",") if len(fields) > sep + 3 else []}) return entries @@ -832,32 +853,525 @@ def seal(work): return 0 -def enumerate_sockets(budget=180): - """Every socket file on the local file systems (find -xdev per mount) - and every bound pathname socket of this network namespace - (/proc/net/unix, what ss -xl shows). Raises when the budget runs out.""" - start = time.monotonic() +def proc_net_unix_paths(): + """The bound pathname sockets of this network namespace (/proc/net/unix, + what ss -xl shows).""" found = set() - points = sorted({m["point"] for m in mounts() if m["fstype"] not in PSEUDO}) + with open("/proc/net/unix", "rb") as f: + next(f) + for line in f: + parts = line.split(None, 7) + if len(parts) == 8 and parts[7].startswith(b"/"): + found.add(parts[7].rstrip(b"\n")) + return found + + +def enumerate_sockets(budget=180, clock=time.monotonic, run=subprocess.run, entries=None): + """The walk: every socket file on the local file systems (find -xdev per + mount, as the calling user) and every bound pathname socket of this + network namespace. Raises when the budget runs out.""" + start = clock() + found = set() + points = sorted({m["point"] for m in (mounts() if entries is None else entries) if m["fstype"] not in PSEUDO}) for p in points: - left = budget - (time.monotonic() - start) + left = budget - (clock() - start) if left <= 0: - raise TimeoutError(f"the {budget}s budget ran out before {p}") + raise TimeoutError(f"the {budget:g}s budget ran out before {p}") try: - r = subprocess.run(["find", p, "-xdev", "-type", "s", "-print0"], stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, timeout=left) + r = run(["find", p, "-xdev", "-type", "s", "-print0"], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, timeout=left) except subprocess.TimeoutExpired: - raise TimeoutError(f"the {budget}s budget ran out in {p}") from None + raise TimeoutError(f"the {budget:g}s budget ran out in {p}") from None if r.returncode not in (0, 1): # 1: some directories were unreadable raise OSError(f"find {p} exited with {r.returncode}") found.update(x for x in r.stdout.split(b"\0") if x) - with open("/proc/net/unix", "rb") as f: - next(f) + found |= proc_net_unix_paths() + return sorted(found), len(points), clock() - start + + +# The fast listing (--sockets-fast, as root, outside the namespaces). A +# pathname Unix socket accepts a connection only while a socket is bound to +# its inode, and connecting finds that socket by inode whatever network +# namespace it is in; a read-only mount does not stop it. So what must not +# be reachable inside is every live, bound pathname socket of every network +# namespace, at every path the mount tree leads to its inode by. The kernel +# lists them (sock_diag, UNIX_DIAG_VFS: the device and inode of the bound +# file). The listing is used only if it is shown complete: every Unix +# socket any process holds (fd tables, io_uring tables) is in a listed +# network namespace, and every bound socket is found again at its name as +# the same inode with one link on a local file system, so its paths are the +# mounts of its file system that contain it. Anything else, any error and +# running out of time (Incomplete) make no-docker.sh run the walk. + +NLMSG_ERROR, NLMSG_DONE = 2, 3 +NLM_F_REQUEST, NLM_F_DUMP, NLM_F_DUMP_INTR = 0x1, 0x300, 0x10 +NETLINK_SOCK_DIAG, SOCK_DIAG_BY_FAMILY = 4, 20 +UDIAG_SHOW_NAME, UDIAG_SHOW_VFS = 0x1, 0x2 +UNIX_DIAG_NAME, UNIX_DIAG_VFS = 0, 1 +LINUX_AF_UNIX, LINUX_MSG_TRUNC, LINUX_CLONE_NEWNET = 1, 0x20, 0x40000000 +# File systems whose link counts and inode numbers are the kernel's own. +# On FUSE, NFS, 9p, CIFS, virtiofs or overlayfs a server or the layers +# decide them, so one link does not mean one name there: the walk runs. +LOCAL_FS = {"ext2", "ext3", "ext4", "xfs", "btrfs", "tmpfs", "devtmpfs", "hugetlbfs"} +FAST_BUDGET = 10.0 + + +class Incomplete(Exception): + """The fast listing cannot show that it saw every live socket.""" + + +class NoEnd(Incomplete): + """The socket dump has not ended yet (more replies to read).""" + + +class Deadline: + def __init__(self, seconds, clock=time.monotonic): + self.seconds, self.clock = seconds, clock + self.end = clock() + seconds + + def left(self): + return self.end - self.clock() + + def check(self, what): + if self.left() <= 0: + raise Incomplete(f"the {self.seconds:g}s budget ran out {what}") + + +def nl_align(n): + return (n + 3) & ~3 + + +def diag_request(seq): + """A dump request for every AF_UNIX socket in every state, with its name + and the device and inode of its file.""" + req = struct.pack(" len(payload): + raise Incomplete(f"socket {ino}: an attribute of length {alen} overruns its record") + body = payload[off + 4:off + alen] + if atype == UNIX_DIAG_NAME: + name = bytes(body) + elif atype == UNIX_DIAG_VFS: + if len(body) != 8: + raise Incomplete(f"socket {ino}: a file attribute of {len(body)} bytes") + vfs = struct.unpack(" len(data): + raise Incomplete(f"a netlink message of length {length} overruns its reply") + if mseq != seq: + raise Incomplete(f"a netlink message of sequence {mseq}, not {seq}") + if mflags & NLM_F_DUMP_INTR: + raise Incomplete("the socket dump was interrupted") + if mtype == NLMSG_DONE: + if off + nl_align(length) < len(data) or n != len(batches) - 1: + raise Incomplete("data after the end of the socket dump") + return out + if mtype == NLMSG_ERROR: + code = struct.unpack_from("= 20 else 0 + raise Incomplete(f"the socket dump failed ({os.strerror(-code) if code < 0 else code})") + if mtype != SOCK_DIAG_BY_FAMILY: + raise Incomplete(f"a netlink message of type {mtype}") + out.append(parse_diag_msg(data[off + 16:off + length])) + off += nl_align(length) + raise NoEnd("the socket dump has no end marker") + + +def kernel_dev(dev): + """(major, minor) of a kernel dev_t (s_dev, as sock_diag reports it).""" + return dev >> 20, dev & 0xFFFFF + + +def unescape_bytes(s): + return re.sub(rb"\\([0-7]{3})", lambda m: bytes([int(m.group(1), 8)]), s) + + +def parse_mountinfo(data): + """mountinfo as bytes: id, (major, minor), root and mount point (bytes), + fstype.""" + entries = [] + for line in data.split(b"\n"): + if not line: + continue + f = line.split(b" ") + sep = f.index(b"-") + major, minor = (int(x) for x in f[2].split(b":")) + entries.append({"id": int(f[0]), "dev": (major, minor), "root": unescape_bytes(f[3]), + "point": unescape_bytes(f[4]), "fstype": f[sep + 1].decode("ascii", "replace")}) + return entries + + +def under(path, top): + """True if path is top or below it, by whole path components.""" + return top == b"/" or path == top or path.startswith(top.rstrip(b"/") + b"/") + + +def rebase(path, old, new): + """path (under old) moved from old to new.""" + rest = path if old == b"/" else path[len(old):] + if not rest: + return new + return (b"" if new == b"/" else new.rstrip(b"/")) + rest + + +def fs_path(root, point, canonical): + """The path within its file system of the file at canonical, reached + through the mount of root at point; None if canonical is not below it.""" + if not under(canonical, point): + return None + return rebase(canonical, point, root) + + +def aliases(entries, dev, path): + """Every path through which a mount of the file system dev leads to the + file at path within it (bind mounts of the file or of a directory above + it included, hidden ones too).""" + return sorted({rebase(path, m["root"], m["point"]) for m in entries + if m["dev"] == dev and under(path, m["root"])}) + + +def locate_reason(mode, ino, nlink, vfs, mount, canonical): + """None if the file opened at a socket's name is that socket's bound file + with one name, else why not.""" + vino, vdev = vfs + if not stat.S_ISSOCK(mode): + return "its name is not a socket now" + if ino & 0xFFFFFFFF != vino: + return "its name is another inode now" + if nlink != 1: + return f"its file has {nlink} links" + if mount is None: + return "its mount is not in mountinfo" + if mount["dev"] != kernel_dev(vdev): + return f"its name is on device {mount['dev'][0]}:{mount['dev'][1]}, not {kernel_dev(vdev)[0]}:{kernel_dev(vdev)[1]}" + if mount["fstype"] not in LOCAL_FS: + return f"it is on {mount['fstype']}" + if canonical.endswith(b" (deleted)"): + return "its file was removed" + return None + + +def claim(seen, key, vfs): + """Record that the socket vfs is located at key ((dev, path)); two live + sockets located at one path mean one of them is elsewhere.""" + if key in seen and seen[key] != vfs: + raise Incomplete(f"two live sockets are located at {key[1]!r}") + seen[key] = vfs + + +def cross_check(held, dumped): + """The Unix sockets held by processes that no dump listed.""" + return sorted({i & 0xFFFFFFFF for i in held} - {i & 0xFFFFFFFF for i in dumped}) + + +def uring_sockets(fdinfo): + """The sockets in the registered file table of an io_uring (its fdinfo). + The kernel leaves the table out when its lock is busy: Incomplete.""" + head, sep, rest = fdinfo.partition("\nUserFiles:") + if not sep: + raise Incomplete("an io_uring's registered files are not shown") + table = re.split(r"\n(?=\S)", rest, maxsplit=1)[0] + return {int(x) for x in re.findall(r"^\s*\d+: socket:\[(\d+)\]$", table, re.M)} + + +def gone(e): + """True for the errors of a process or fd that ended meanwhile.""" + return e.errno in (errno.ENOENT, errno.ESRCH) + + +def proc_entries(path): + """The numeric entries of a /proc directory; [] if the process ended.""" + try: + return [d for d in os.listdir(path) if d.isdigit()] + except OSError as e: + if gone(e): + return [] + raise Incomplete(f"{path}: {e.strerror}") from None + + +def proc_readlink(path): + try: + return os.readlink(path) + except OSError as e: + if gone(e): + return None + raise Incomplete(f"{path}: {e.strerror}") from None + + +def scan_tasks(deadline): + """Every network namespace with a task, a bind mount or an open fd + ({inode: [(path, nofollow)]}), the Unix sockets held in fd tables and in + io_uring tables, and whether pid 1 was seen.""" + nets, held, uring, mntns = {}, set(), set(), set() + pid1 = False + for pid in proc_entries("/proc"): + deadline.check("while scanning the processes") + # An exited thread group leader has no namespace links, but its + # other threads may still run: they are scanned below. + mnt = proc_readlink(f"/proc/{pid}/ns/mnt") + if mnt is not None and mnt not in mntns: + mntns.add(mnt) + try: + with open(f"/proc/{pid}/mountinfo", "rb") as f: + info = parse_mountinfo(f.read()) + except OSError as e: + if not gone(e): + raise Incomplete(f"/proc/{pid}/mountinfo: {e.strerror}") from None + mntns.discard(mnt) + info = [] + for m in info: + r = re.fullmatch(rb"net:\[(\d+)\]", m["root"]) + if m["fstype"] == "nsfs" and r: + nets.setdefault(int(r.group(1)), []).append((f"/proc/{pid}/root".encode() + m["point"], True)) + for tid in proc_entries(f"/proc/{pid}/task"): + base = f"/proc/{pid}/task/{tid}" + net = proc_readlink(f"{base}/ns/net") + r = re.fullmatch(r"net:\[(\d+)\]", net or "") + if r: + nets.setdefault(int(r.group(1)), []).append((f"{base}/ns/net".encode(), False)) + for fd in proc_entries(f"{base}/fd"): + link = proc_readlink(f"{base}/fd/{fd}") or "" + if link.startswith("socket:["): + try: + proto = os.getxattr(f"{base}/fd/{fd}", "system.sockprotoname") + except OSError as e: + if gone(e): + continue + raise Incomplete(f"{base}/fd/{fd}: {e.strerror}") from None + if proto.startswith(b"UNIX"): + held.add(int(link[8:-1])) + elif re.fullmatch(r"net:\[\d+\]", link): + nets.setdefault(int(link[5:-1]), []).append((f"{base}/fd/{fd}".encode(), False)) + elif link == "anon_inode:[io_uring]": + try: + with open(f"{base}/fdinfo/{fd}") as f: + uring |= uring_sockets(f.read()) + except OSError as e: + if not gone(e): + raise Incomplete(f"{base}/fdinfo/{fd}: {e.strerror}") from None + pid1 = pid1 or pid == "1" + return nets, held, uring, pid1 + + +def setns(fd, nstype): + if hasattr(os, "setns"): + os.setns(fd, nstype) + return + libc = ctypes.CDLL(None, use_errno=True) + if libc.setns(fd, nstype) != 0: + raise OSError(ctypes.get_errno(), "setns failed") + + +def open_netns(path, nofollow, ino): + """An fd of network namespace ino, opened through path only once the + object there is shown to be that namespace (O_PATH opens nothing).""" + fd = os.open(path, os.O_PATH | os.O_CLOEXEC | (os.O_NOFOLLOW if nofollow else 0)) + try: + if os.readlink(f"/proc/self/fd/{fd}") != f"net:[{ino}]": + raise OSError(errno.EINVAL, f"not network namespace {ino}") + return os.open(f"/proc/self/fd/{fd}", os.O_RDONLY | os.O_CLOEXEC) + finally: + os.close(fd) + + +def diag_dump(timeout): + """Every Unix socket of this network namespace (sock_diag).""" + s = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW | socket.SOCK_CLOEXEC, NETLINK_SOCK_DIAG) + try: + s.settimeout(timeout) + s.bind((0, 0)) + s.sendto(diag_request(1), (0, 0)) + batches = [] + while True: + data, _, flags, addr = s.recvmsg(1 << 17) + batches.append((data, flags, addr[0])) + try: + return parse_diag(batches, 1) + except NoEnd: + continue + finally: + s.close() + + +def dump_netns(ino, fd, deadline): + """diag_dump in network namespace ino, in a child process the deadline + kills: setns stays confined to it, and a blocked call cannot hold up + the listing.""" + r, w = os.pipe() + child = os.fork() + if child == 0: + try: + os.close(r) + setns(fd, LINUX_CLONE_NEWNET) + if os.readlink("/proc/thread-self/ns/net") != f"net:[{ino}]": + raise Incomplete("setns did not enter the namespace") + recs = diag_dump(max(deadline.left(), 0.1)) + out = {"complete": True, "records": [[x["ino"], x["path"].hex() if x["path"] is not None else None, + list(x["vfs"]) if x["vfs"] else None] for x in recs]} + except BaseException as e: # reported to the parent, which falls back + out = {"error": f"{type(e).__name__}: {e}"[:300]} + data = json.dumps(out).encode() + try: + while data: + data = data[os.write(w, data):] + finally: + os._exit(0) + os.close(w) + chunks = [] + try: + while True: + left = deadline.left() + if left <= 0 or not select.select([r], [], [], left)[0]: + os.kill(child, signal.SIGKILL) + raise Incomplete(f"the {deadline.seconds:g}s budget ran out in network namespace {ino}") + chunk = os.read(r, 1 << 16) + if not chunk: + break + chunks.append(chunk) + finally: + os.close(r) + os.waitpid(child, 0) + try: + out = json.loads(b"".join(chunks)) + except ValueError: + raise Incomplete(f"no result from network namespace {ino}") from None + if out.get("complete") is not True: + raise Incomplete(f"network namespace {ino}: {out.get('error', 'no result')}") + return [{"ino": i, "path": bytes.fromhex(p) if p is not None else None, "vfs": tuple(v) if v else None} + for i, p, v in out["records"]] + + +def mnt_id_of_fd(fd): + with open(f"/proc/self/fdinfo/{fd}") as f: for line in f: - parts = line.split(None, 7) - if len(parts) == 8 and parts[7].startswith(b"/"): - found.add(parts[7].rstrip(b"\n")) - return sorted(found), len(points), time.monotonic() - start + if line.startswith("mnt_id:"): + return int(line.split()[1]) + raise Incomplete("no mnt_id in fdinfo") + + +def locate(rec, host): + """(path within its file system, mount) of a bound socket's file, found + again at its name; Incomplete if that cannot be shown.""" + name = rec["path"] + what = f"socket {name!r} (dev {kernel_dev(rec['vfs'][1])[0]}:{kernel_dev(rec['vfs'][1])[1]}, ino {rec['vfs'][0]})" + if not name.startswith(b"/"): + raise Incomplete(f"{what}: a relative name") + try: + fd = os.open(name, os.O_PATH | os.O_NOFOLLOW | os.O_CLOEXEC) + except OSError as e: + raise Incomplete(f"{what}: {e.strerror}") from None + try: + st = os.fstat(fd) + mnt = mnt_id_of_fd(fd) + canonical = os.readlink(f"/proc/self/fd/{fd}".encode()) + finally: + os.close(fd) + mount = next((m for m in host if m["id"] == mnt), None) + why = locate_reason(st.st_mode, st.st_ino, st.st_nlink, rec["vfs"], mount, canonical) + if why: + raise Incomplete(f"{what}: {why}") + path = fs_path(mount["root"], mount["point"], canonical) + if path is None: + raise Incomplete(f"{what}: {canonical!r} is not below its mount point") + return path, mount + + +def fast_listing(deadline, paths): + """Fill paths with every path to every live bound pathname socket, and + return the counts for the log; Incomplete if that cannot be shown.""" + if os.geteuid() != 0: + raise Incomplete("not run as root") + if os.readlink("/proc/self/ns/pid") != INIT_PID_NS: + raise Incomplete("not in the initial pid namespace") + if os.readlink("/proc/self") != str(os.getpid()): + raise Incomplete("/proc is not of this pid namespace") + home = os.readlink("/proc/thread-self/ns/net") + with open("/proc/self/mountinfo", "rb") as f: + host = parse_mountinfo(f.read()) + for attempt in (1, 2): + nets, held, uring, pid1 = scan_tasks(deadline) + dumped = {} + for ino, handles in sorted(nets.items()): + deadline.check("before the socket dumps") + errors = [] + for path, nofollow in handles: + try: + fd = open_netns(path, nofollow, ino) + except OSError as e: + errors.append(f"{path!r}: {e.strerror or e}") + continue + try: + dumped[ino] = dump_netns(ino, fd, deadline) + finally: + os.close(fd) + break + else: + raise Incomplete(f"network namespace {ino} could not be entered ({'; '.join(errors[:3])})") + missing = cross_check(held | uring, [x["ino"] for recs in dumped.values() for x in recs]) + if not missing: + break + if attempt == 2: + raise Incomplete(f"{len(missing)} Unix sockets held by processes are in no listed network namespace " + f"(socket:[{missing[0]}])") + if not held or not pid1: + raise Incomplete("the process scan saw no Unix socket or not pid 1") + if not dumped.get(int(home[5:-1])): + raise Incomplete("the host network namespace listed no Unix socket") + seen = {} + live = bound = 0 + for recs in dumped.values(): + for rec in recs: + live += 1 + if rec["path"] is None: + continue + bound += 1 + deadline.check("while locating the bound sockets") + path, mount = locate(rec, host) + claim(seen, (mount["dev"], path), rec["vfs"]) + paths.update(aliases(host, mount["dev"], path)) + if os.readlink("/proc/thread-self/ns/net") != home: + raise Incomplete("not in the host network namespace any more") + paths.update(proc_net_unix_paths()) + return {"namespaces": len(dumped), "live": live, "bound": bound, "held": len(held | uring)} def unix_connects(path): @@ -895,96 +1409,219 @@ def env_pairs(name): return dict(x.split("=", 1) for x in os.environ.get(name, "").split() if "=" in x) -def check_mechanism_linux(): - me = os.getuid() - st = status() +def eval_creds(st, group): + """Findings on this process's credentials (st: /proc/self/status).""" uids = st.get("Uid", "0").split() - report("0" not in uids, f"not root (uids {' '.join(uids)})") caps = {k: st.get(k, "unknown") for k in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb")} - report(all(v != "unknown" and int(v, 16) == 0 for v in caps.values()), - "no capabilities (" + " ".join(f"{k}={v}" for k, v in caps.items()) + ")") - report(st.get("NoNewPrivs") == "1", f"no_new_privs is set ({st.get('NoNewPrivs', 'unknown')})") - report(st.get("Groups", "x") == "", f"no supplementary groups ({st.get('Groups') or 'none'})") - # The primary group is the caller's (on a runner possibly the engine - # socket's group): what keeps the socket away is that its path is - # hidden and that no listed socket accepts a connection (below). - print(f"no-docker: info uids {st.get('Uid', '?')}; gids {st.get('Gid', '?')}") - - host_ns = env_pairs("NO_DOCKER_HOST_NS") - for ns in ("net", "mnt", "pid", "ipc"): - own = os.readlink(f"/proc/self/ns/{ns}") - host = host_ns.get(ns, "") - report(bool(host) and own != host, f"private {ns} namespace ({own}; caller {host or 'unknown'})") - nspid = st.get("NSpid", "").split() - report(len(nspid) >= 2, f"in a pid namespace below the caller's (NSpid {' '.join(nspid)})") - report(os.readlink("/proc/self") == str(os.getpid()), "/proc belongs to this pid namespace") + + def zero(v): + return re.fullmatch(r"[0-9a-f]+", v) is not None and int(v, 16) == 0 + + gids = st.get("Gid", "").split() + plain = len(gids) == 4 and len(set(gids)) == 1 and gids[0].isdigit() and int(gids[0]) > SYS_GID_MAX + return [ + ("0" not in uids, f"not root (uids {' '.join(uids)})"), + (all(zero(v) for v in caps.values()), "no capabilities (" + " ".join(f"{k}={v}" for k, v in caps.items()) + ")"), + (st.get("NoNewPrivs") == "1", f"no_new_privs is set ({st.get('NoNewPrivs', 'unknown')})"), + (st.get("Groups", "x") == "", f"no supplementary groups ({st.get('Groups') or 'none'})"), + # The primary group is the caller's: not one of the system groups, + # which include the one of the engine socket. + (plain, f"the primary group is not a system group (gids {' '.join(gids) or 'unknown'}; {group})"), + ] + + +def eval_pidns(nspid, pid, own, host_ns, proc_self): + """Findings on the pid namespace. host_ns: NO_DOCKER_HOST_NS as given.""" + callers = [x[len("pid="):] for x in host_ns.split() if x.startswith("pid=")] + caller = callers[0] if len(callers) == 1 else "" + well_formed = re.fullmatch(r"pid:\[\d+\]", own) is not None + return [ + (caller == INIT_PID_NS, f"the caller is in the initial pid namespace (caller {caller or 'unknown'})"), + # Every pid namespace descends from the initial one: one that is + # not it is below the caller's. + (well_formed and own != INIT_PID_NS and own != caller, + f"in a pid namespace below the caller's ({own}; caller {caller or 'unknown'})"), + # A procfs shows NSpid from its own namespace down: one entry, this + # pid, in the namespace's own /proc; more from an ancestor's. + (nspid.split() == [str(pid)], f"/proc shows this process in its own pid namespace only (NSpid {nspid or 'missing'})"), + (proc_self == str(pid), "/proc belongs to this pid namespace"), + ] + + +def eval_processes(entries, me): + """entries: (pid, its status or the OSError reading it) for every pid in + /proc. Only an ended process may be skipped: one that cannot be read + (hidepid) counts as foreign, and pid 1 must have been read.""" foreign = [] count = 0 - for d in os.listdir("/proc"): - if not d.isdigit(): + root = False + for d, st in entries: + if isinstance(st, OSError): + if not gone(st): + foreign.append(f"{d}:unreadable({errno.errorcode.get(st.errno, st.errno)})") continue - try: - pst = status(d) - except OSError: - continue # ended meanwhile count += 1 - name, puids = pst.get("Name", "?"), pst.get("Uid", "").split() + name, puids = st.get("Name", "?"), st.get("Uid", "").split() if d == "1": - if puids[:1] != ["0"] or name not in ("sh", "dash"): + root = puids[:1] == ["0"] and name in ("sh", "dash") + if not root: foreign.append(f"1:{name}:{'/'.join(puids)}") - elif any(u != str(me) for u in puids) or name in HOST_COMMS: + elif not puids or any(u != str(me) for u in puids) or name in HOST_COMMS: foreign.append(f"{d}:{name}:{'/'.join(puids)}") - report(not foreign, f"only our processes and the root pid 1 of the isolation are visible ({count} processes" - + (f"; foreign: {' '.join(foreign)}" if foreign else "") + ")") + if not root and not any(f.startswith("1:") for f in foreign): + foreign.append("1:missing") + return [(not foreign, f"only our processes and the root pid 1 of the isolation are visible ({count} processes" + + (f"; foreign: {' '.join(foreign)}" if foreign else "") + ")")] + + +def eval_mounts(vis, host_devs, devs, work, left): + """vis: the reachable mounts by mount point; devs: st_dev of the private + paths; left: the mounts still writable (None without a run directory).""" + f = [] + proc = vis.get("/proc") + hidden = [o for o in (proc or {}).get("opts", []) + (proc or {}).get("super", []) + if (o.startswith("hidepid=") and o not in ("hidepid=0", "hidepid=off")) or o.startswith("subset=")] + f.append((bool(proc) and proc["fstype"] == "proc" and not hidden, + f"/proc shows every process of the namespace ({','.join(hidden) or 'no hidepid or subset'})")) + for path, fstype in PRIVATE: + m = vis.get(path) + dev = devs.get(path, "?") + host = host_devs.get(path, "") + f.append((bool(m) and m["fstype"] == fstype and bool(host) and dev != host, + f"{path} is a private {fstype} (device {dev}; caller {host or 'unknown'})")) + # A second procfs, or a namespace file, reachable by path would lead to + # the host's processes (/proc//root) or namespaces. + extra = [f"{m['point']} ({m['fstype']})" for m in vis.values() + if (m["fstype"] == "proc" and m["point"] != "/proc") or m["fstype"] == "nsfs"] + f.append((not extra, "no other proc or nsfs mount is reachable" + (f" ({' '.join(extra)})" if extra else ""))) + m = vis.get("/sys/fs/cgroup") + f.append((bool(m) and m["fstype"] == "tmpfs" and "ro" in m["opts"], "/sys/fs/cgroup is masked by a read-only tmpfs")) + if work: + f.append((not left, f"every other mount is read-only ({len(vis)} reachable mounts" + + (f"; writable: {' '.join(left)}" if left else "") + ")")) + return f + + +def eval_stdio(kinds): + """kinds: (fd, kind) of fds 0 to 2. A Unix socket there could reach a + host service without a connect.""" + return [(all(k != "unix socket" for _, k in kinds), + "no standard descriptor is a Unix socket (" + ", ".join(f"{fd}: {k}" for fd, k in kinds) + ")")] + + +def eval_reached(paths, reached): + """An empty list proves nothing: it fails.""" + return [(bool(paths) and not reached, f"none of the {len(paths)} Unix sockets listed outside accepts a connection" + + (": " + ", ".join(reached) if reached else ""))] + + +def evaluate_linux(o): + """Every finding of the mechanism check, from the observations o.""" + f = eval_creds(o["status"], o["group"]) + for ns in ("net", "mnt", "pid", "ipc"): + own, host = o["ns"][ns], o["host_ns"].get(ns, "") + f.append((bool(host) and own != host, f"private {ns} namespace ({own}; caller {host or 'unknown'})")) + f += eval_pidns(o["status"].get("NSpid", ""), o["pid"], o["ns"]["pid"], o["host_ns_raw"], o["proc_self"]) + f += eval_processes(o["processes"], o["uid"]) + f.append((o["sid_ours"], f"a session of our own (session {o['sid']})")) + f.append((o["tty"] != "opened", f"no controlling terminal ({o['tty']})")) + f += eval_stdio(o["stdio"]) + f.append((o["interfaces"] == ["lo"], f"network namespace has only loopback ({' '.join(o['interfaces'])})")) + f.append((bool(o["work"]), f"the run's directory is known ({o['work'] or 'NO_DOCKER_RUN_DIR unset'})")) + f += eval_mounts(o["mounts"], o["host_devs"], o["devs"], o["work"], o["writable"]) + f.append((o["cwd_reachable"] is True, f"the working directory is reachable by its path ({o['cwd']}" + + ("" if o["cwd_reachable"] in (True, False) else f": {o['cwd_reachable']}") + ")")) + return f + + +def observe_linux(): + """What the mechanism check looks at, read once (no write, no connect).""" + st = status() + try: + group = grp.getgrgid(os.getgid()).gr_name + except KeyError: + group = "no group name" + processes = [] + for d in os.listdir("/proc"): + if d.isdigit(): + try: + processes.append((d, status(d))) + except OSError as e: + processes.append((d, e)) sid = os.getsid(0) sid_ours = False if sid > 1: try: - sid_ours = all(u == str(me) for u in status(sid).get("Uid", "").split()) + sid_ours = all(u == str(os.getuid()) for u in status(sid).get("Uid", "").split()) except OSError: pass - report(sid_ours, f"a session of our own (session {sid})") try: os.close(os.open("/dev/tty", os.O_RDWR)) - report(False, "no controlling terminal (TIOCSTI)") + tty = "opened" except OSError as e: - report(True, f"no controlling terminal ({errno.errorcode.get(e.errno, e.errno)})") - - names = sorted(n for _, n in socket.if_nameindex()) - report(names == ["lo"], f"network namespace has only loopback ({' '.join(names)})") - + tty = errno.errorcode.get(e.errno, str(e.errno)) + stdio = [] + for fd in (0, 1, 2): + try: + mode = os.fstat(fd).st_mode + except OSError: + stdio.append((fd, "closed")) + continue + kind = "socket" if stat.S_ISSOCK(mode) else "pipe" if stat.S_ISFIFO(mode) else "character device" \ + if stat.S_ISCHR(mode) else "file" if stat.S_ISREG(mode) else "other" + if kind == "socket": + s = socket.socket(fileno=os.dup(fd)) + kind = "unix socket" if s.family == socket.AF_UNIX else "socket" + s.close() + stdio.append((fd, kind)) work = os.environ.get("NO_DOCKER_RUN_DIR", "") - report(bool(work), f"the run's directory is known ({work or 'NO_DOCKER_RUN_DIR unset'})") - host_devs = env_pairs("NO_DOCKER_HOST_DEVS") vis = {m["point"]: m for m in visible_mounts()} - for path, fstype in PRIVATE: - m = vis.get(path) + devs = {} + for path, _ in PRIVATE: try: - dev = str(os.stat(path).st_dev) + devs[path] = str(os.stat(path).st_dev) except OSError: - dev = "?" - host = host_devs.get(path, "") - report(bool(m) and m["fstype"] == fstype and bool(host) and dev != host, - f"{path} is a private {fstype} (device {dev}; caller {host or 'unknown'})") - # A second procfs, or a namespace file, reachable by path would lead to - # the host's processes (/proc//root) or namespaces. - extra = [f"{m['point']} ({m['fstype']})" for m in vis.values() - if (m["fstype"] == "proc" and m["point"] != "/proc") or m["fstype"] == "nsfs"] - report(not extra, "no other proc or nsfs mount is reachable" + (f" ({' '.join(extra)})" if extra else "")) - m = vis.get("/sys/fs/cgroup") - report(bool(m) and m["fstype"] == "tmpfs" and "ro" in m["opts"], "/sys/fs/cgroup is masked by a read-only tmpfs") - if work: - left = writable_mounts(work) - report(not left, f"every other mount is read-only ({len(vis)} reachable mounts" - + (f"; writable: {' '.join(left)}" if left else "") + ")") + devs[path] = "?" try: - report(mount_id_of(".") == mount_id_of(os.getcwd()), f"the working directory is reachable by its path ({os.getcwd()})") + cwd_reachable = mount_id_of(".") == mount_id_of(os.getcwd()) except OSError as e: - report(False, f"the working directory is reachable by its path ({e})") - if not ok: - # Not isolated: write nothing and connect to nothing. - return + cwd_reachable = str(e) + return { + "status": st, "group": group, "uid": os.getuid(), "pid": os.getpid(), "proc_self": os.readlink("/proc/self"), + "ns": {ns: os.readlink(f"/proc/self/ns/{ns}") for ns in ("net", "mnt", "pid", "ipc")}, + "host_ns": env_pairs("NO_DOCKER_HOST_NS"), "host_ns_raw": os.environ.get("NO_DOCKER_HOST_NS", ""), + "processes": processes, "sid": sid, "sid_ours": sid_ours, "tty": tty, "stdio": stdio, + "interfaces": sorted(n for _, n in socket.if_nameindex()), "work": work, "mounts": vis, + "host_devs": env_pairs("NO_DOCKER_HOST_DEVS"), "devs": devs, + "writable": writable_mounts(work) if work else None, "cwd": os.getcwd(), "cwd_reachable": cwd_reachable, + } + +def gated(observe, evaluate, after, say): + """Report every finding, and only if there are findings and all passed + run after (the part that writes and connects). Any error before that is + a failure.""" + try: + findings = evaluate(observe()) + except Exception as e: + say(False, f"the isolation check failed: {type(e).__name__}: {e}") + return False + if not findings: + say(False, "the isolation check found nothing to check") + return False + good = True + for passed, text in findings: + say(passed, text) + good = good and passed + if not good: + return False + after() + return True + + +def after_gate_linux(): + """Isolated: writes must fail, no cgroup file is writable, no socket in + the private directories, and no socket listed outside connects.""" protected = [p for p in os.environ.get("NO_DOCKER_PROTECTED", "").split(":") if p] report(bool(protected), "the protected directories are known (NO_DOCKER_PROTECTED)") for p in protected: @@ -1014,8 +1651,13 @@ def check_mechanism_linux(): return with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: reached = [f"{p} ({k})" for p, k in zip(paths, pool.map(unix_connects, paths)) if k] - report(not reached, f"none of the {len(paths)} Unix sockets listed outside accepts a connection" - + (": " + ", ".join(reached) if reached else "")) + for passed, text in eval_reached(paths, reached): + report(passed, text) + + +def check_mechanism_linux(): + print(f"no-docker: info uids {status().get('Uid', '?')}; gids {status().get('Gid', '?')}") + gated(observe_linux, evaluate_linux, after_gate_linux, report) def listening_ports(strict=False): @@ -1122,14 +1764,30 @@ def main(): return 1 print("\n".join(sorted({str(port) for _, port in ports}, key=int))) return 0 - if sys.argv[1:] == ["--sockets"]: + if sys.argv[1:2] == ["--sockets"] and len(sys.argv) <= 3: try: - paths, points, took = enumerate_sockets() - except (OSError, TimeoutError, StopIteration) as e: + paths, points, took = enumerate_sockets(float(sys.argv[2]) if len(sys.argv) == 3 else 180) + except (OSError, TimeoutError, StopIteration, ValueError) as e: print(f"no-docker: cannot list the Unix sockets: {e}", file=sys.stderr) return 1 sys.stdout.buffer.write(b"".join(p + b"\0" for p in paths)) - print(f"no-docker: {len(paths)} Unix sockets listed on {points} mounts in {took:.1f}s", file=sys.stderr) + print(f"no-docker: {len(paths)} Unix sockets listed on {points} mounts in {took:.1f}s (mode walk)", file=sys.stderr) + return 0 + if sys.argv[1:] == ["--sockets-fast"] and sys.platform.startswith("linux"): + # Exit 0: the complete list. Exit 3: not shown complete; the paths + # located so far are printed and no-docker.sh runs the walk. + start = time.monotonic() + paths = set() + try: + counts = fast_listing(Deadline(FAST_BUDGET), paths) + except Exception as e: # any doubt: the walk runs + sys.stdout.buffer.write(b"".join(p + b"\0" for p in sorted(paths))) + print(f"no-docker: listing mode walk: {type(e).__name__}: {e}", file=sys.stderr) + return 3 + sys.stdout.buffer.write(b"".join(p + b"\0" for p in sorted(paths))) + print(f"no-docker: {len(paths)} Unix sockets listed (mode fast; {counts['namespaces']} network namespaces, " + f"{counts['live']} live, {counts['bound']} bound to a path, {counts['held']} held by processes) " + f"in {time.monotonic() - start:.1f}s", file=sys.stderr) return 0 if sys.platform == "darwin" and (sys.argv[1:2] == ["--markers"] and len(sys.argv) == 4 or sys.argv[1:2] in (["--orphans"], ["--end"]) and len(sys.argv) == 3): diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh index eaaa020..7b459a7 100755 --- a/scripts/ci/no-docker.sh +++ b/scripts/ci/no-docker.sh @@ -96,7 +96,7 @@ fi [ $# -gt 0 ] || refuse "no command given" if [ -n "${NO_DOCKER_ACTIVE-}" ]; then # Already isolated (a suite re-executing itself): never nest. - python3 -I "$probe" >/dev/null || refuse "inside no-docker.sh, but the isolation check fails" + python3 -I "$probe" >&2 || refuse "inside no-docker.sh, but the isolation check fails" [ -z "$given_work" ] || refuse "--work cannot be used inside the isolation" exec "$@" fi @@ -247,6 +247,10 @@ case $os in n=$((n + 1)) done [ "$found" = yes ] || refuse "Linux is supported on GitHub-hosted CI runners only (no Runner.Worker ancestor)" + # The new pid namespace is shown to be below the caller's by the + # caller's being the initial one (no-docker-probe.py). + [ "$(readlink /proc/self/ns/pid)" = "pid:[4026531836]" ] || + refuse "Linux is supported on GitHub-hosted CI runners only (not in the initial pid namespace)" protected=$home for v in GITHUB_WORKSPACE RUNNER_TEMP; do d=$(cd -P "${!v:-/nonexistent}" 2>/dev/null && pwd -P) || refuse "Linux is supported on GitHub-hosted CI runners only ($v is not a directory)" @@ -254,7 +258,7 @@ case $os in case $d in "$home"/?*) ;; *) refuse "Linux is supported on GitHub-hosted CI runners only ($v is not under \$HOME)" ;; esac protected="$protected:$d" done - for tool in sudo unshare setpriv setsid ip mount find stat python3; do + for tool in sudo timeout unshare setpriv setsid ip mount find stat python3; do command -v "$tool" >/dev/null || refuse "$tool not found" done sudo -n true 2>/dev/null || refuse "creating the namespaces needs passwordless sudo" @@ -278,6 +282,10 @@ case $os in v=$(stat -c %d "$d") || refuse "cannot stat $d" host_devs="$host_devs $d=$v" done + # The command keeps the caller's primary group (the probe refuses a + # system group); for the log, next to the group of the engine socket. + printf 'no-docker: info primary group %s (%s); /var/run/docker.sock group %s\n' "$(id -g)" \ + "$(id -gn 2>/dev/null || echo '?')" "$(stat -Lc '%g (%G)' /var/run/docker.sock 2>/dev/null || echo none)" >&2 ;; *) refuse "no isolation mechanism for $os" @@ -372,11 +380,32 @@ case $os in refuse "cannot start the supervisor (python3)" ;; Linux) - # Every Unix socket on the local file systems and every bound one of - # the host's network namespace; inside, none may accept a connection. - # A time budget that runs out refuses rather than truncating the list. - timing=$(python3 -I "$probe" --sockets 2>&1 >"$work/sockets.list") || refuse "cannot list the Unix sockets: $timing" - # Its duration, in the log of every run (the budget is 180 s). + # The Unix sockets that must not accept a connection inside: every live + # pathname socket of every network namespace at every path to it, from + # the kernel (as root, killed after 15 s; it writes nothing, its list + # comes through this shell's redirection). Unless that is shown + # complete (exit 0), the walk of every local file system runs as before, + # as the caller, and both lists are used. 180 s in all; a budget that + # runs out refuses rather than truncating the list. + start=$SECONDS + # shellcheck disable=SC2024 # on purpose: this shell, not root, writes the list + timing=$(sudo -n timeout -s KILL 15 python3 -I "$probe" --sockets-fast 2>&1 >"$work/sockets.fast") + frc=$? + if [ "$frc" -ne 0 ]; then + [ "$frc" -eq 3 ] || { + timing="no-docker: listing mode walk: the fast listing ended with status $frc" + : >"$work/sockets.fast" || refuse "cannot reset the socket list" + } + printf '%s\n' "$timing" >&2 + left=$((180 - (SECONDS - start))) + [ "$left" -gt 0 ] || refuse "cannot list the Unix sockets: the 180s budget ran out" + timing=$(python3 -I "$probe" --sockets "$left" 2>&1 >"$work/sockets.walk") || refuse "cannot list the Unix sockets: $timing" + if ! cat "$work/sockets.walk" >>"$work/sockets.fast" || ! rm -f "$work/sockets.walk"; then + refuse "cannot keep the socket list" + fi + fi + mv "$work/sockets.fast" "$work/sockets.list" || refuse "cannot keep the socket list" + # Its mode and duration, in the log of every run. printf '%s\n' "$timing" >&2 printf '::notice title=no-docker::%s\n' "${timing#no-docker: }" envargs+=("NO_DOCKER_SOCKETS=/run/no-docker/sockets.list" "NO_DOCKER_HOST_NS=${host_ns# }" From f01b8441e6e2356a56456d9a64f4acd48c7b55f6 Mon Sep 17 00:00:00 2001 From: Sebastian Szyja Date: Fri, 30 Oct 2026 23:32:57 +0100 Subject: [PATCH 33/77] test(ci): test the Linux probe's rules purely and refuse a live socket the walk cannot see T-36 runs the probe's Linux rules on fixtures, with every I/O call made to raise, fixed assertion counts and eleven mutants that must be rejected. T-37 binds a listener in an unlistable directory, in the host network namespace and in one of its own, which must be refused; T-02 requires the refusal to come from the mechanism check, and T-16 checks NSpid as the pid alone, the pid namespace's descent and /proc options, with an exact count. --- scripts/ci/tests/no-docker-probe-unit.py | 495 +++++++++++++++++++++++ scripts/ci/tests/no-docker.test.sh | 145 ++++++- 2 files changed, 623 insertions(+), 17 deletions(-) create mode 100644 scripts/ci/tests/no-docker-probe-unit.py diff --git a/scripts/ci/tests/no-docker-probe-unit.py b/scripts/ci/tests/no-docker-probe-unit.py new file mode 100644 index 0000000..bd9738d --- /dev/null +++ b/scripts/ci/tests/no-docker-probe-unit.py @@ -0,0 +1,495 @@ +"""Pure tests of the Linux rules of scripts/ci/no-docker-probe.py. + +Usage: python3 -I -B no-docker-probe-unit.py + +The probe's source is read once, then every call that would read /proc, +open a file or socket, run a program, fork or enter a namespace is replaced +by one that raises: the rules are tested on fixtures only, never on this +machine. Each assertion prints "ok " or "FAIL "; +the cases are C01 to C10 (the regression cases of FCRM-SEC-019), W (the +probe calls the tested functions) and M1 to M11 (mutated copies of the +probe, each of which must fail at least one assertion of its case). The +last line is "SUMMARY ok, failed". Exit 0 only if nothing +failed; 2 if a mutant cannot be made exactly. +""" + +import ast +import builtins +import errno +import os +import socket +import stat +import struct +import subprocess +import sys + +sys.dont_write_bytecode = True +SOURCE_PATH = sys.argv[1] +with open(SOURCE_PATH) as f: + SOURCE = f.read() + + +def load(src): + ns = {"__name__": "probe_under_test", "__file__": SOURCE_PATH} + exec(compile(src, SOURCE_PATH, "exec"), ns) + return ns + + +class IOAttempt(AssertionError): + pass + + +def tripwire(name): + def raiser(*args, **kwargs): + raise IOAttempt(f"{name} called in a pure test") + return raiser + + +for mod, names in ((os, ("open", "listdir", "readlink", "getxattr", "setns", "fork", "stat", "lstat", "fstat", + "scandir", "walk", "pipe", "kill", "waitpid")), + (socket, ("socket", "socketpair", "if_nameindex")), + (subprocess, ("run", "Popen", "check_output", "call"))): + for name in names: + if hasattr(mod, name): + setattr(mod, name, tripwire(f"{mod.__name__}.{name}")) +builtins.open = tripwire("open") + +P = load(SOURCE) +INIT = "pid:[4026531836]" +HOST_NS = f"net=net:[4026531833] mnt=mnt:[4026531832] pid={INIT} ipc=ipc:[4026531839]" +results = {"ok": 0, "fail": 0} + + +def emit(tag, good, text): + results["ok" if good else "fail"] += 1 + print(f"{'ok ' if good else 'FAIL'} {tag} {text}") + + +def passes(findings): + return bool(findings) and all(p for p, _ in findings) + + +def failing(findings): + return [t for p, t in findings if not p] + + +def status(**over): + st = {"Name": "python3", "Uid": "1001\t1001\t1001\t1001", "Gid": "1001\t1001\t1001\t1001", "Groups": "", + "NoNewPrivs": "1", "NSpid": "20", "CapInh": "0000000000000000", "CapPrm": "0000000000000000", + "CapEff": "0000000000000000", "CapBnd": "0000000000000000", "CapAmb": "0000000000000000"} + st.update(over) + return {k: v for k, v in st.items() if v is not None} + + +def mount(point, fstype, opts=("rw",), sup=("rw",)): + return {"id": hash(point) & 0xFFFF, "point": point, "fstype": fstype, "opts": list(opts), "super": list(sup)} + + +def valid_mounts(): + vis = {p: mount(p, t) for p, t in (("/tmp", "tmpfs"), ("/var/tmp", "tmpfs"), ("/dev/shm", "tmpfs"), + ("/run", "tmpfs"), ("/dev/mqueue", "mqueue"))} + vis["/proc"] = mount("/proc", "proc") + vis["/sys/fs/cgroup"] = mount("/sys/fs/cgroup", "tmpfs", ("ro",)) + vis["/"] = mount("/", "ext4", ("ro",)) + return vis + + +HOST_DEVS = {"/tmp": "2049", "/var/tmp": "2049", "/dev/shm": "27", "/run": "29", "/dev/mqueue": "23"} +DEVS = {"/tmp": "54", "/var/tmp": "55", "/dev/shm": "56", "/run": "53", "/dev/mqueue": "51"} +PROCS = [("1", {"Name": "sh", "Uid": "0\t0\t0\t0"}), ("20", status()), ("21", status(Name="bash"))] + + +def valid_obs(**over): + o = {"status": status(), "group": "runner", "uid": 1001, "pid": 20, "proc_self": "20", + "ns": {"net": "net:[4026532270]", "mnt": "mnt:[4026532267]", "pid": "pid:[4026532269]", + "ipc": "ipc:[4026532268]"}, + "host_ns": {"net": "net:[4026531833]", "mnt": "mnt:[4026531832]", "pid": INIT, "ipc": "ipc:[4026531839]"}, + "host_ns_raw": HOST_NS, "processes": list(PROCS), "sid": 17, "sid_ours": True, "tty": "ENXIO", + "stdio": [(0, "pipe"), (1, "pipe"), (2, "pipe")], "interfaces": ["lo"], "work": "/tmp/no-docker.x", + "mounts": valid_mounts(), "host_devs": dict(HOST_DEVS), "devs": dict(DEVS), "writable": [], + "cwd": "/home/runner/work", "cwd_reachable": True} + o.update(over) + return o + + +# --- the cases; each takes the probe namespace and an emit function -------- + +def c01(P, emit): + f = P["eval_pidns"]("20", 20, "pid:[4026532269]", HOST_NS, "20") + emit("C01", passes(f) and len(f) == 4, f"a private /proc with NSpid of one entry, this pid, passes ({failing(f)})") + f = P["evaluate_linux"](valid_obs()) + emit("C01", passes(f), f"the whole mechanism check passes on a valid observation ({failing(f)})") + + +def c02(P, emit): + variants = [ + ("NSpid of two entries (a /proc of an ancestor namespace)", ("4711\t20", 20, "pid:[4026532269]", HOST_NS, "20")), + ("NSpid of one entry that is not this pid", ("21", 20, "pid:[4026532269]", HOST_NS, "20")), + ("NSpid missing", ("", 20, "pid:[4026532269]", HOST_NS, "20")), + ("NSpid malformed", ("20 x", 20, "pid:[4026532269]", HOST_NS, "20")), + ("own pid namespace is the caller's", ("20", 20, INIT, HOST_NS, "20")), + ("caller not in the initial pid namespace", ("20", 20, "pid:[4026532269]", HOST_NS.replace(INIT, "pid:[4026532001]"), "20")), + ("own namespace is the initial one, the caller claimed another", ("20", 20, INIT, HOST_NS.replace(INIT, "pid:[4026532001]"), "20")), + ("caller given twice", ("20", 20, "pid:[4026532269]", HOST_NS + f" pid={INIT}", "20")), + ("caller a near miss", ("20", 20, "pid:[4026532269]", HOST_NS.replace(INIT, "pid:[04026531836]"), "20")), + ("caller missing", ("20", 20, "pid:[4026532269]", "net=net:[1]", "20")), + ("own namespace malformed", ("20", 20, "pid:[x]", HOST_NS, "20")), + ("/proc/self is another pid", ("20", 20, "pid:[4026532269]", HOST_NS, "4711")), + ] + for text, args in variants: + f = P["eval_pidns"](*args) + emit("C02", not passes(f), f"rejected: {text} ({len(failing(f))} failing)") + f = P["evaluate_linux"](valid_obs(ns=dict(valid_obs()["ns"], pid=INIT))) + emit("C02", not passes(f), "rejected by the whole check: a pid namespace shared with the caller") + + +def c03(P, emit): + ev = P["eval_processes"] + emit("C03", passes(ev(PROCS, 1001)), "our processes and the root shell as pid 1 pass") + emit("C03", passes(ev(PROCS + [("22", OSError(errno.ENOENT, "gone"))], 1001)), "a process that ended is skipped") + emit("C03", passes(ev(PROCS + [("22", OSError(errno.ESRCH, "gone"))], 1001)), "a process that ended (ESRCH) is skipped") + bad = [ + ("a process of another user", PROCS + [("30", status(Uid="0\t0\t0\t0", Name="sleep"))]), + ("a host daemon under our uid", PROCS + [("31", status(Name="dockerd"))]), + ("Runner.Worker visible", PROCS + [("32", status(Name="Runner.Worker"))]), + ("pid 1 is systemd", [("1", {"Name": "systemd", "Uid": "0\t0\t0\t0"})] + PROCS[1:]), + ("pid 1 is not root", [("1", {"Name": "sh", "Uid": "1001\t1001\t1001\t1001"})] + PROCS[1:]), + ("pid 1 missing", PROCS[1:]), + ("pid 1 unreadable (hidepid)", [("1", OSError(errno.EPERM, "hidden"))] + PROCS[1:]), + ("a status unreadable (EACCES)", PROCS + [("33", OSError(errno.EACCES, "denied"))]), + ("a process without uids", PROCS + [("34", {"Name": "x"})]), + ] + for text, entries in bad: + emit("C03", not passes(ev(entries, 1001)), f"rejected: {text}") + + +def c04(P, emit): + ev = P["eval_creds"] + emit("C04", passes(ev(status(), "runner")), "no capability, no_new_privs, not root passes") + for cap in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb"): + emit("C04", not passes(ev(status(**{cap: "0000000000000400"}), "runner")), f"rejected: {cap} not zero") + emit("C04", not passes(ev(status(**{cap: None}), "runner")), f"rejected: {cap} missing") + emit("C04", not passes(ev(status(CapEff="zzzz"), "runner")), "rejected: an unparsable capability set") + emit("C04", not passes(ev(status(NoNewPrivs="0"), "runner")), "rejected: no_new_privs unset") + emit("C04", not passes(ev(status(Uid="1001\t0\t1001\t1001"), "runner")), "rejected: a uid 0") + + +def c05(P, emit): + ev = P["eval_creds"] + for gid in ("1000", "1001"): + emit("C05", passes(ev(status(Gid="\t".join([gid] * 4)), "g")), f"primary gid {gid} passes") + for gid, why in (("999", "the last system gid"), ("118", "the docker group"), ("6", "disk"), ("0", "root")): + emit("C05", not passes(ev(status(Gid="\t".join([gid] * 4)), "g")), f"rejected: primary gid {gid} ({why})") + emit("C05", not passes(ev(status(Gid="1001\t1001\t118\t1001"), "g")), "rejected: gids that differ") + emit("C05", not passes(ev(status(Gid=None), "g")), "rejected: gids missing") + emit("C05", not passes(ev(status(Groups="118"), "g")), "rejected: a supplementary group") + + +def c06(P, emit): + ev = P["eval_mounts"] + emit("C06", passes(ev(valid_mounts(), HOST_DEVS, DEVS, "/w", [])), "private mounts, one procfs, masked cgroups pass") + v = valid_mounts() + v["/proc"]["super"] = ["rw", "hidepid=0"] + emit("C06", passes(ev(v, HOST_DEVS, DEVS, "/w", [])), "hidepid=0 passes") + for text, change in ( + ("a second procfs", lambda v: v.update({"/mnt/p": mount("/mnt/p", "proc")})), + ("an nsfs file", lambda v: v.update({"/x": mount("/x", "nsfs")})), + ("/proc with hidepid=2", lambda v: v["/proc"].update(super=["rw", "hidepid=2"])), + ("/proc with hidepid=invisible", lambda v: v["/proc"].update(super=["rw", "hidepid=invisible"])), + ("/proc with subset=pid", lambda v: v["/proc"].update(opts=["rw", "subset=pid"])), + ("/proc not a procfs", lambda v: v["/proc"].update(fstype="tmpfs")), + ("/proc missing", lambda v: v.pop("/proc")), + ("/tmp not private", lambda v: v["/tmp"].update(fstype="ext4")), + ("/sys/fs/cgroup writable", lambda v: v["/sys/fs/cgroup"].update(opts=["rw"]))): + v = valid_mounts() + change(v) + emit("C06", not passes(ev(v, HOST_DEVS, DEVS, "/w", [])), f"rejected: {text}") + emit("C06", not passes(ev(valid_mounts(), HOST_DEVS, dict(DEVS, **{"/tmp": "2049"}), "/w", [])), + "rejected: /tmp is the caller's device") + emit("C06", not passes(ev(valid_mounts(), HOST_DEVS, DEVS, "/w", ["/home"])), "rejected: a writable mount left") + + +def c07(P, emit): + ev = P["eval_reached"] + emit("C07", passes(ev(["/a", "/b"], [])), "listed sockets that do not connect pass") + emit("C07", not passes(ev(["/a", "/b"], ["/b (stream)"])), "rejected: a listed socket accepts a connection") + emit("C07", not passes(ev([], [])), "rejected: an empty list proves nothing") + emit("C07", not passes(P["eval_stdio"]([(0, "unix socket"), (1, "pipe"), (2, "pipe")])), + "rejected: a standard descriptor is a Unix socket") + emit("C07", passes(P["eval_stdio"]([(0, "character device"), (1, "pipe"), (2, "pipe")])), "pipes and /dev/null pass") + + +# netlink fixtures: the layout of include/uapi/linux/{netlink,sock_diag,unix_diag}.h +def nlmsg(mtype, payload, seq=1, flags=0x2): + return struct.pack("