diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 95dad26ab..15c80def1 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,5 @@ -FROM ghcr.io/framework-r-d/phlex-dev:2026-07-23 +ARG PHLEX_DEV_IMAGE=ghcr.io/framework-r-d/phlex-dev:latest +FROM ${PHLEX_DEV_IMAGE} # Validate Python site-packages symlink. # Invoke bash explicitly: the script uses bash features (pipefail, arrays, @@ -103,7 +104,7 @@ INSTALL_DEV_UTILS RUN bash <<'INSTALL_NPM_PACKAGES' set -euo pipefail -npm update -g npm@^11.10.0 +npm install -g npm@^11.10.0 npm config set allow-scripts=@kilocode/cli --location=user npm config set min-release-age=3 --location=user npm install -g @kilocode/cli @@ -117,9 +118,12 @@ RUN printf '. /entrypoint.sh\n' >> /root/.bashrc # allow GPG agent to function correctly when running as root. RUN mkdir -p /run/user/0 && chmod 700 /run/user/0 && chown root:root /run/user/0 -COPY gitignore-global /root/.gitignore-global +COPY .devcontainer/gitignore-global /root/.gitignore-global RUN git config --global core.excludesfile '/root/.gitignore-global' # Install Kilo environment script that sets KILO_CONFIG_CONTENT globally -COPY kilo-env.sh /etc/profile.d/kilo-env.sh -RUN chmod +x /etc/profile.d/kilo-env.sh +# Keep runtime fixes current without rebuilding the expensive Spack base image. +COPY ci/entrypoint.sh /entrypoint.sh +COPY .devcontainer/kilo-env.sh /etc/profile.d/kilo-env.sh +COPY .devcontainer/kilo-env-self-test.sh /usr/local/lib/phlex/kilo-env-self-test.sh +RUN chmod +x /etc/profile.d/kilo-env.sh /usr/local/lib/phlex/kilo-env-self-test.sh diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 000000000..3d08cbaa2 --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,21 @@ +# Native macOS Devcontainer + +The initializer generates the gitignored `docker-compose.host.yml` before VS Code resolves Compose. On Apple Silicon it requires a running rootless arm64 Podman VM and a locally available Linux/arm64 image, `localhost/phlex-dev:latest` by default. Set `PHLEX_DEV_IMAGE` in the environment launching VS Code to select another image reference. A multi-arch tag or manifest is supported when Podman can resolve its Linux/arm64 member from local storage; the generated Compose service explicitly selects `linux/arm64`. The initializer does not query remote manifests or pull automatically: for a registry image, explicitly run `podman pull --platform linux/arm64 ` first. A manifest listing an arm64 member is not sufficient unless that member is available locally. Linux retains the Dockerfile's `ghcr.io/framework-r-d/phlex-dev:latest` default and the engine's native platform selection. + +macOS uses pasta networking. Ordinary development does not mount the Podman engine socket or disable SELinux container labeling. Host Kilo configuration and auth inputs are read-only; auth is copied into the container-private data volume when first created. + +## Optional Nested Podman + +For future in-container `act` use, fully quit VS Code Insiders first, then launch a fresh process from the repository root: + +```bash +PHLEX_ENABLE_NESTED_PODMAN=1 code-insiders . +``` + +Run **Dev Containers: Rebuild and Reopen in Container**. The initializer then mounts the VM's rootless Unix socket at `/tmp/podman.sock`, sets `DOCKER_HOST` and `CONTAINER_HOST`, and adds `security_opt: ["label=disable"]`. This explicitly reduces isolation: the devcontainer can control all containers and resources managed by the VM's rootless engine. The read-only bind mount does not make API operations read-only. No TCP API or macOS Unix-socket relay is used. + +To restore the default boundary, quit Insiders, launch it without `PHLEX_ENABLE_NESTED_PODMAN`, and rebuild the devcontainer. Do not edit the generated override; initialization replaces it on each launch. + +## Self-tests + +`kilo-env-self-test.sh` and `ensure-repos.sh` execute their own unit tests when the `--self-test` option is provided. Fixture listeners, assertions, and test cases are loaded by sourcing the appropriate `*-self-test.sh` file, and the tests executed in lieu of each script executing its production code. diff --git a/.devcontainer/configure-host.py b/.devcontainer/configure-host.py new file mode 100644 index 000000000..47181cd0c --- /dev/null +++ b/.devcontainer/configure-host.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Persist host choices for Compose; initializeCommand exports cannot reach VS Code.""" + +from __future__ import annotations + +import json +import os +import platform +import subprocess +import tempfile +from pathlib import Path + + +def host_service(system: str, home: Path) -> dict: + """Use VM paths on macOS, never Unix sockets on the macOS shared filesystem.""" + service: dict = {"volumes": []} + image = os.environ.get("PHLEX_DEV_IMAGE") + if system == "Darwin": + info = subprocess.check_output( + [ + "podman", + "info", + "--format", + "{{.Host.Arch}} {{.Host.Security.Rootless}} {{.Host.RemoteSocket.Path}}", + ], + text=True, + ).split() + if len(info) != 3 or info[:2] != ["arm64", "true"]: + raise ValueError( + "macOS development requires a running native arm64 rootless Podman VM" + ) + socket = info[2].removeprefix("unix://") + if not socket.startswith("/run/user/") or not socket.endswith("/podman/podman.sock"): + raise ValueError(f"Unexpected rootless VM socket: {socket}") + image = image or "localhost/phlex-dev:latest" + # Podman resolves local manifest lists to their native-platform member. + image_platform = subprocess.check_output( + ["podman", "image", "inspect", "--format", "{{.Os}}/{{.Architecture}}", image], + text=True, + ).strip() + if image_platform != "linux/arm64": + raise ValueError( + f"{image} does not resolve to a local linux/arm64 image; " + "build natively or explicitly pull its linux/arm64 member first" + ) + service["platform"] = "linux/arm64" + service["network_mode"] = "pasta" + if os.environ.get("PHLEX_ENABLE_NESTED_PODMAN") == "1": + # Explicitly opt in to engine control and reduced SELinux isolation. + service["volumes"].append(f"{socket}:/tmp/podman.sock:ro") + service["security_opt"] = ["label=disable"] + else: + proxy_socket = home / ".podman-proxy/podman.sock" + if proxy_socket.exists(): + service["volumes"].append(f"{proxy_socket}:/tmp/podman.sock:Z") + if service["volumes"]: + service["environment"] = { + "DOCKER_HOST": "unix:///tmp/podman.sock", + "CONTAINER_HOST": "unix:///tmp/podman.sock", + } + if image: + service["build"] = {"args": {"PHLEX_DEV_IMAGE": image}} + auth = home / ".local/share/kilo/auth.json" + if auth.is_file(): + service["volumes"].append(f"{auth}:/run/phlex-host-auth.json:ro,Z") + return service + + +def main() -> None: + """Atomically replace the generated, gitignored Compose override.""" + target = Path(__file__).with_name("docker-compose.host.yml") + service = host_service(platform.system(), Path.home()) + # JSON is also YAML, and safely quotes paths containing spaces or punctuation. + fd, temporary = tempfile.mkstemp(prefix=".compose-host-", dir=target.parent) + try: + with os.fdopen(fd, "w") as output: + output.write(json.dumps({"services": {"phlex-dev": service}}, indent=2) + "\n") + os.replace(temporary, target) + finally: + Path(temporary).unlink(missing_ok=True) + + +if __name__ == "__main__": + main() diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ce8ebdc57..af90e7b00 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,9 @@ { "name": "Phlex CI Dev Container", - "dockerComposeFile": "docker-compose.yml", + "dockerComposeFile": [ + "docker-compose.yml", + "docker-compose.host.yml" + ], "service": "phlex-dev", "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", "remoteUser": "root", @@ -25,7 +28,6 @@ "C_Cpp.default.cStandard": "c17", "C_Cpp.default.compileCommands": "${workspaceFolder}/build/compile_commands.json", "C_Cpp.default.cppStandard": "c++23", - "C_Cpp.default.intelliSenseMode": "linux-gcc-x64", "files.watcherExclude": { "**/.venv/**": true, "**/py_virtual_env/**": true diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 129ac8909..e2c6d571a 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -3,8 +3,8 @@ version: "3.8" services: phlex-dev: build: - context: . - dockerfile: Dockerfile + context: .. + dockerfile: .devcontainer/Dockerfile volumes: - ..:/workspaces/phlex:Z - ../../phlex-coding-guidelines:/workspaces/phlex-coding-guidelines:Z @@ -12,18 +12,16 @@ services: - ../../phlex-examples:/workspaces/phlex-examples:Z - ../../phlex-spack-recipes:/workspaces/phlex-spack-recipes:Z - ${HOME}/.config/gh:/root/.config/gh:ro,Z - - ${HOME}/.config/kilo:/root/.config/kilo:Z + - ${HOME}/.config/kilo:/root/.config/kilo:ro,Z + - ${PHLEX_RELAY_STATE_DIR:-${HOME}/.phlex-devcontainer-tmp/relays}:/run/phlex-host-relays:ro,Z - ${HOME}/.gnupg:/root/.gnupg:ro,Z - - ${HOME}/.podman-proxy/podman.sock:/tmp/podman.sock:Z - phlex-kilo-data:/root/.local/share/kilo:Z - - ${HOME}/.local/share/kilo/auth.json:/root/.local/share/kilo/auth.json:ro,Z - phlex-vscode-user-data:/root/.vscode-server-insiders/data/User:Z environment: - CMAKE_GENERATOR=Ninja - GH_CONFIG_DIR=/root/.config/gh - - DOCKER_HOST=unix:///tmp/podman.sock - - CONTAINER_HOST=unix:///tmp/podman.sock - GNUPGHOME=/root/.gnupg + - PHLEX_DEFAULT_COMPILER=${PHLEX_DEFAULT_COMPILER:-clang} user: root tty: true stdin_open: true diff --git a/.devcontainer/ensure-repos-self-test.sh b/.devcontainer/ensure-repos-self-test.sh new file mode 100644 index 000000000..b569b8335 --- /dev/null +++ b/.devcontainer/ensure-repos-self-test.sh @@ -0,0 +1,474 @@ +#!/bin/bash + +# Helper functions for self-test mode - sourced, not executed +# This file contains test-only utilities and the mode_self_test implementation +# It must be sourced by ensure-repos.sh, not executed directly + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + printf '%s\n' 'ERROR: ensure-repos-self-test.sh must be sourced, not executed' >&2 + exit 1 +fi + +file_sha256() { + local path="$1" + + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$path" | awk '{print $1}' + elif command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{print $1}' + else + return 1 + fi +} + +start_python_loopback_listener() { + local label="$2" + local port="$1" + + python3 - "$port" "$label" <<'PY' >/dev/null 2>&1 & +import signal +import socket +import sys + +port = int(sys.argv[1]) +label = sys.argv[2].encode("utf-8") + b"\n" +stop = False + +def handle_signal(signum, frame): + del signum, frame + global stop + stop = True + +signal.signal(signal.SIGTERM, handle_signal) +signal.signal(signal.SIGINT, handle_signal) + +sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +sock.bind(("127.0.0.1", port)) +sock.listen(16) +sock.settimeout(0.2) + +while not stop: + try: + conn, _ = sock.accept() + except socket.timeout: + continue + except OSError: + break + + with conn: + try: + conn.sendall(label) + except OSError: + pass + +sock.close() +PY + + printf '%s\n' "$!" +} + +probe_python_tcp_message() { + local expected_label="$2" + local port="$1" + + python3 - "$port" "$expected_label" <<'PY' +import socket +import sys + +port = int(sys.argv[1]) +expected = (sys.argv[2] + "\n").encode("utf-8") + +sock = socket.create_connection(("127.0.0.1", port), timeout=2.0) +try: + data = sock.recv(128) +finally: + sock.close() + +if data != expected: + raise SystemExit(1) +PY +} + +# mode_self_test_loaded implements the actual self-test logic +# This function must be called from mode_self_test() in ensure-repos.sh +mode_self_test_loaded() { + local PHLEX_RELAY_ALLOWED_RANGE="127.0.0.0/8" + local default_env="${DEFAULT_PHLEX_RELAY_STATE_DIR}/relay-map.env" + local default_env_hash_before="" + local default_env_missing_before=0 + local default_json="${DEFAULT_PHLEX_RELAY_STATE_DIR}/relay-map.json" + local default_json_hash_before="" + local default_json_missing_before=0 + local test_failed=0 + + SELF_TEST_BLOCKER_PID="" + SELF_TEST_LISTENER_25114_PID="" + SELF_TEST_LISTENER_25300_PID="" + SELF_TEST_ROGUE_PID="" + SELF_TEST_SAVED_HOST_PORTS="" + SELF_TEST_SAVED_HOST_PORTS_SET=0 + SELF_TEST_SAVED_TEST_PORTS="" + SELF_TEST_SAVED_TEST_PORTS_SET=0 + if [ "${PHLEX_HOST_RELAY_PORTS+x}" = x ]; then + SELF_TEST_SAVED_HOST_PORTS="$PHLEX_HOST_RELAY_PORTS" + SELF_TEST_SAVED_HOST_PORTS_SET=1 + fi + if [ "${PHLEX_TEST_RELAY_PORTS+x}" = x ]; then + SELF_TEST_SAVED_TEST_PORTS="$PHLEX_TEST_RELAY_PORTS" + SELF_TEST_SAVED_TEST_PORTS_SET=1 + fi + + assert_file_content() { + local actual="" + local expected="$1" + local label="$3" + local path="$2" + + if [ -f "$path" ]; then + actual=$(cat "$path") + fi + + if [ "$actual" != "$expected" ]; then + echo "ERROR: ${label} mismatch" >&2 + echo " expected: $expected" >&2 + echo " actual: $actual" >&2 + return 1 + fi + + return 0 + } + + assert_empty_state() { + local pidfile + local pidfile_count=0 + + assert_file_content '{}' "$RELAY_MAP_JSON" 'empty relay-map.json' || return 1 + [ ! -s "$RELAY_MAP_ENV" ] || { + echo "ERROR: expected empty relay-map.env" >&2 + return 1 + } + + for pidfile in "$RELAY_PID_DIR"/*; do + [ -e "$pidfile" ] || continue + case "$(basename "$pidfile")" in + '' | *[!0-9]*) + continue + ;; + *) + pidfile_count=$((pidfile_count + 1)) + ;; + esac + done + if [ "$pidfile_count" -ne 0 ]; then + echo "ERROR: expected no numeric relay PID files, found $pidfile_count" >&2 + return 1 + fi + + if detect_numeric_relay_listener 25115 || detect_numeric_relay_listener 25301; then + echo "ERROR: expected no ready numeric relay listeners" >&2 + return 1 + fi + + return 0 + } + + restore_saved_environment() { + if [ "$SELF_TEST_SAVED_HOST_PORTS_SET" -eq 0 ]; then + unset PHLEX_HOST_RELAY_PORTS + else + PHLEX_HOST_RELAY_PORTS="$SELF_TEST_SAVED_HOST_PORTS" + fi + + if [ "$SELF_TEST_SAVED_TEST_PORTS_SET" -eq 0 ]; then + unset PHLEX_TEST_RELAY_PORTS + else + PHLEX_TEST_RELAY_PORTS="$SELF_TEST_SAVED_TEST_PORTS" + fi + } + + self_test_cleanup() { + mode_cleanup_owned >/dev/null 2>&1 || true + + if [ -n "$SELF_TEST_BLOCKER_PID" ] && kill -0 "$SELF_TEST_BLOCKER_PID" 2>/dev/null; then + terminate_pid_gracefully "$SELF_TEST_BLOCKER_PID" >/dev/null 2>&1 || true + fi + SELF_TEST_BLOCKER_PID="" + + if [ -n "$SELF_TEST_ROGUE_PID" ] && kill -0 "$SELF_TEST_ROGUE_PID" 2>/dev/null; then + terminate_pid_gracefully "$SELF_TEST_ROGUE_PID" >/dev/null 2>&1 || true + fi + SELF_TEST_ROGUE_PID="" + + if [ -n "$SELF_TEST_LISTENER_25114_PID" ] && kill -0 "$SELF_TEST_LISTENER_25114_PID" 2>/dev/null; then + terminate_pid_gracefully "$SELF_TEST_LISTENER_25114_PID" >/dev/null 2>&1 || true + fi + SELF_TEST_LISTENER_25114_PID="" + + if [ -n "$SELF_TEST_LISTENER_25300_PID" ] && kill -0 "$SELF_TEST_LISTENER_25300_PID" 2>/dev/null; then + terminate_pid_gracefully "$SELF_TEST_LISTENER_25300_PID" >/dev/null 2>&1 || true + fi + SELF_TEST_LISTENER_25300_PID="" + + restore_saved_environment + } + + trap 'self_test_cleanup' EXIT + trap 'trap - EXIT; self_test_cleanup; exit 129' HUP + trap 'trap - EXIT; self_test_cleanup; exit 130' INT + trap 'trap - EXIT; self_test_cleanup; exit 143' TERM + + mkdir -p "$PHLEX_RELAY_STATE_DIR" "$RELAY_PID_DIR" + write_relay_maps + + if [ -f "$default_json" ]; then + default_json_hash_before=$(file_sha256 "$default_json") || return 1 + else + default_json_missing_before=1 + fi + + if [ -f "$default_env" ]; then + default_env_hash_before=$(file_sha256 "$default_env") || return 1 + else + default_env_missing_before=1 + fi + + SELF_TEST_LISTENER_25114_PID=$(start_python_loopback_listener 25114 25114) + SELF_TEST_LISTENER_25300_PID=$(start_python_loopback_listener 25300 25300) + wait_for_source_listener_ready 25114 || { + echo "ERROR: source listener 25114 did not become ready" >&2 + return 1 + } + wait_for_source_listener_ready 25300 || { + echo "ERROR: source listener 25300 did not become ready" >&2 + return 1 + } + + echo "Self-test: ready named services" + PHLEX_TEST_RELAY_PORTS="$RELAY_ALLOWLIST" + if ! mode_start_test_services; then + echo "ERROR: --start-test-services positive path failed" >&2 + test_failed=1 + fi + assert_file_content '{"25114":"25115","25300":"25301"}' "$RELAY_MAP_JSON" 'ready relay-map.json' || test_failed=1 + assert_file_content $'PHLEX_HOST_RELAY_25114=25115\nPHLEX_HOST_RELAY_25300=25301' "$RELAY_MAP_ENV" 'ready relay-map.env' || test_failed=1 + probe_python_tcp_message 25115 25114 || { + echo "ERROR: relay 25115 did not reach source 25114" >&2 + test_failed=1 + } + probe_python_tcp_message 25301 25300 || { + echo "ERROR: relay 25301 did not reach source 25300" >&2 + test_failed=1 + } + + local first_pid_25114="" + local first_pid_25300="" + first_pid_25114=$(cat "$(relay_pid_file 25114)" 2>/dev/null || true) + first_pid_25300=$(cat "$(relay_pid_file 25300)" 2>/dev/null || true) + [[ "$first_pid_25114" =~ ^[0-9]+$ ]] || { + echo "ERROR: missing numeric PID file for 25114" >&2 + test_failed=1 + } + [[ "$first_pid_25300" =~ ^[0-9]+$ ]] || { + echo "ERROR: missing numeric PID file for 25300" >&2 + test_failed=1 + } + + echo "Self-test: repeat convergence" + if ! mode_start_test_services; then + echo "ERROR: repeat --start-test-services failed" >&2 + test_failed=1 + fi + if [ "$first_pid_25114" != "$(cat "$(relay_pid_file 25114)" 2>/dev/null || true)" ]; then + echo "ERROR: relay 25114 did not converge on the same owned PID" >&2 + test_failed=1 + fi + if [ "$first_pid_25300" != "$(cat "$(relay_pid_file 25300)" 2>/dev/null || true)" ]; then + echo "ERROR: relay 25300 did not converge on the same owned PID" >&2 + test_failed=1 + fi + + echo "Self-test: deterministic ordering" + PHLEX_HOST_RELAY_PORTS="25300=25301,25114=25115" + if ! mode_default; then + echo "ERROR: reverse-order relay admission failed" >&2 + test_failed=1 + fi + assert_file_content '{"25114":"25115","25300":"25301"}' "$RELAY_MAP_JSON" 'sorted relay-map.json' || test_failed=1 + assert_file_content $'PHLEX_HOST_RELAY_25114=25115\nPHLEX_HOST_RELAY_25300=25301' "$RELAY_MAP_ENV" 'sorted relay-map.env' || test_failed=1 + + echo "Self-test: positive cleanup" + if ! mode_cleanup_owned; then + echo "ERROR: cleanup of owned relays failed" >&2 + test_failed=1 + fi + assert_empty_state || test_failed=1 + + echo "Self-test: listener-start failure" + SELF_TEST_BLOCKER_PID=$(start_python_loopback_listener 25115 blocker) + wait_for_source_listener_ready 25115 || { + echo "ERROR: blocking listener on 25115 did not become ready" >&2 + return 1 + } + PHLEX_HOST_RELAY_PORTS="$RELAY_ALLOWLIST" + if mode_default; then + echo "ERROR: expected listener-start failure when 25115 is already listening" >&2 + test_failed=1 + fi + assert_file_content '{"25300":"25301"}' "$RELAY_MAP_JSON" 'partial relay-map.json after listener-start failure' || test_failed=1 + assert_file_content 'PHLEX_HOST_RELAY_25300=25301' "$RELAY_MAP_ENV" 'partial relay-map.env after listener-start failure' || test_failed=1 + [ ! -f "$(relay_pid_file 25114)" ] || { + echo "ERROR: failed relay 25114 left a PID file behind" >&2 + test_failed=1 + } + probe_python_tcp_message 25301 25300 || { + echo "ERROR: surviving relay 25301 did not reach source 25300" >&2 + test_failed=1 + } + terminate_pid_gracefully "$SELF_TEST_BLOCKER_PID" >/dev/null 2>&1 || { + echo "ERROR: failed to stop blocking listener on 25115" >&2 + test_failed=1 + } + SELF_TEST_BLOCKER_PID="" + if ! mode_cleanup_owned; then + echo "ERROR: cleanup after listener-start failure failed" >&2 + test_failed=1 + fi + assert_empty_state || test_failed=1 + + echo "Self-test: malformed input" + PHLEX_HOST_RELAY_PORTS="25114=25115,garbage" + if mode_default; then + echo "ERROR: malformed relay input unexpectedly succeeded" >&2 + test_failed=1 + fi + assert_empty_state || test_failed=1 + + echo "Self-test: duplicate input" + PHLEX_HOST_RELAY_PORTS="25114=25115,25114=25115" + if mode_default; then + echo "ERROR: duplicate relay input unexpectedly succeeded" >&2 + test_failed=1 + fi + assert_empty_state || test_failed=1 + + echo "Self-test: privileged input" + PHLEX_HOST_RELAY_PORTS="25114=25115,80=8080" + if mode_default; then + echo "ERROR: privileged relay input unexpectedly succeeded" >&2 + test_failed=1 + fi + assert_empty_state || test_failed=1 + + echo "Self-test: equal-port input" + PHLEX_HOST_RELAY_PORTS="25114=25115,25300=25300" + if mode_default; then + echo "ERROR: equal-port relay input unexpectedly succeeded" >&2 + test_failed=1 + fi + assert_empty_state || test_failed=1 + + echo "Self-test: unlisted input" + PHLEX_HOST_RELAY_PORTS="25114=25115,25300=25302" + if mode_default; then + echo "ERROR: unlisted relay input unexpectedly succeeded" >&2 + test_failed=1 + fi + assert_empty_state || test_failed=1 + + echo "Self-test: unavailable source input" + if ! terminate_pid_gracefully "$SELF_TEST_LISTENER_25300_PID" >/dev/null 2>&1; then + echo "ERROR: failed to stop source listener 25300 for unavailable-source test" >&2 + test_failed=1 + fi + SELF_TEST_LISTENER_25300_PID="" + PHLEX_HOST_RELAY_PORTS="$RELAY_ALLOWLIST" + if mode_default; then + echo "ERROR: unavailable source relay input unexpectedly succeeded" >&2 + test_failed=1 + fi + assert_empty_state || test_failed=1 + SELF_TEST_LISTENER_25300_PID=$(start_python_loopback_listener 25300 25300) + wait_for_source_listener_ready 25300 || { + echo "ERROR: source listener 25300 did not restart" >&2 + return 1 + } + + echo "Self-test: cleanup ownership boundary" + SELF_TEST_ROGUE_PID=$(start_python_loopback_listener 26001 rogue) + wait_for_source_listener_ready 26001 || { + echo "ERROR: rogue process listener 26001 did not become ready" >&2 + return 1 + } + printf '%s\n' "$SELF_TEST_ROGUE_PID" > "$(relay_pid_file 25114)" + if ! mode_cleanup_owned; then + echo "ERROR: cleanup-owned failed to discard a stale PID file" >&2 + test_failed=1 + fi + if ! kill -0 "$SELF_TEST_ROGUE_PID" 2>/dev/null; then + echo "ERROR: cleanup-owned terminated an unrelated process" >&2 + test_failed=1 + fi + [ ! -f "$(relay_pid_file 25114)" ] || { + echo "ERROR: cleanup-owned retained a stale PID file" >&2 + test_failed=1 + } + assert_empty_state || test_failed=1 + + echo "Self-test: startup after PID reuse" + printf '%s\n' "$SELF_TEST_ROGUE_PID" > "$(relay_pid_file 25114)" + PHLEX_HOST_RELAY_PORTS="$RELAY_ALLOWLIST" + if ! mode_default; then + echo "ERROR: startup failed to recover from a stale PID file" >&2 + test_failed=1 + fi + probe_python_tcp_message 25115 25114 || test_failed=1 + probe_python_tcp_message 25301 25300 || test_failed=1 + if ! kill -0 "$SELF_TEST_ROGUE_PID" 2>/dev/null; then + echo "ERROR: startup terminated an unrelated process" >&2 + test_failed=1 + fi + mode_cleanup_owned || test_failed=1 + terminate_pid_gracefully "$SELF_TEST_ROGUE_PID" >/dev/null 2>&1 || { + echo "ERROR: failed to stop rogue process listener 26001" >&2 + test_failed=1 + } + SELF_TEST_ROGUE_PID="" + assert_empty_state || test_failed=1 + + if [ "$default_json_missing_before" -eq 1 ]; then + [ ! -e "$default_json" ] || { + echo "ERROR: self-test created production relay-map.json" >&2 + test_failed=1 + } + else + [ "$(file_sha256 "$default_json")" = "$default_json_hash_before" ] || { + echo "ERROR: self-test modified production relay-map.json" >&2 + test_failed=1 + } + fi + + if [ "$default_env_missing_before" -eq 1 ]; then + [ ! -e "$default_env" ] || { + echo "ERROR: self-test created production relay-map.env" >&2 + test_failed=1 + } + else + [ "$(file_sha256 "$default_env")" = "$default_env_hash_before" ] || { + echo "ERROR: self-test modified production relay-map.env" >&2 + test_failed=1 + } + fi + + restore_saved_environment + trap - EXIT HUP INT TERM + self_test_cleanup + + if [ "$test_failed" -ne 0 ]; then + echo "Self-test FAILED" >&2 + return 1 + fi + + echo "Self-test passed" + return 0 +} diff --git a/.devcontainer/ensure-repos.sh b/.devcontainer/ensure-repos.sh index ef8ea76aa..5b020d240 100755 --- a/.devcontainer/ensure-repos.sh +++ b/.devcontainer/ensure-repos.sh @@ -1,5 +1,10 @@ #!/bin/bash +if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then + printf '%s\n' 'ERROR: ensure-repos.sh must be executed, not sourced' >&2 + return 1 +fi + # Ensure auxiliary repositories exist on the host as siblings of the phlex # repository. Called by devcontainer.json initializeCommand before the # container is created, so that bind mounts always have a valid source path. @@ -16,6 +21,107 @@ set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PARENT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" +# --- CLI options --- +MODE="default" +SELF_TEST_ROOT="" + +set_mode() { + local requested_mode="$1" + + if [ "$MODE" != "default" ] && [ "$MODE" != "$requested_mode" ]; then + echo "ERROR: only one execution mode may be selected" >&2 + return 1 + fi + + MODE="$requested_mode" +} + +parse_cli_args() { + while [ "$#" -gt 0 ]; do + case "$1" in + --start-test-services) + set_mode "start-test-services" || return 1 + shift + ;; + --start-test-services=*) + echo "ERROR: --start-test-services does not accept a value" >&2 + return 1 + ;; + --self-test) + set_mode "self-test" || return 1 + shift + if [ "$#" -gt 0 ] && [[ "$1" != --* ]]; then + SELF_TEST_ROOT="$1" + shift + fi + ;; + --self-test=*) + set_mode "self-test" || return 1 + SELF_TEST_ROOT="${1#*=}" + shift + ;; + --cleanup-owned) + set_mode "cleanup-owned" || return 1 + shift + ;; + *) + echo "ERROR: unknown option: $1" >&2 + return 1 + ;; + esac + done +} + +parse_cli_args "$@" || exit 1 + +# --- Relay configuration --- +DEFAULT_PHLEX_RELAY_STATE_DIR="${HOME}/.phlex-devcontainer-tmp/relays" + +canonical_path() { + python3 - "$1" <<'PY' +import os +import sys + +print(os.path.realpath(sys.argv[1])) +PY +} + +if [ "$MODE" = "self-test" ]; then + if [ -n "${PHLEX_RELAY_STATE_DIR:-}" ]; then + : + elif [ -n "$SELF_TEST_ROOT" ]; then + PHLEX_RELAY_STATE_DIR="${SELF_TEST_ROOT%/}/relay-state" + else + echo "ERROR: --self-test requires PHLEX_RELAY_STATE_DIR or a directory argument" >&2 + exit 1 + fi + + canonical_state_dir=$(canonical_path "$PHLEX_RELAY_STATE_DIR") || exit 1 + canonical_default_state_dir=$(canonical_path "$DEFAULT_PHLEX_RELAY_STATE_DIR") || exit 1 + if [ "$canonical_state_dir" = "$canonical_default_state_dir" ]; then + echo "ERROR: --self-test must not use the production relay state directory" >&2 + exit 1 + fi +else + PHLEX_RELAY_STATE_DIR="${PHLEX_RELAY_STATE_DIR:-$DEFAULT_PHLEX_RELAY_STATE_DIR}" +fi + +mkdir -p "$PHLEX_RELAY_STATE_DIR" + +RELAY_MAP_JSON="${PHLEX_RELAY_STATE_DIR}/relay-map.json" +RELAY_MAP_ENV="${PHLEX_RELAY_STATE_DIR}/relay-map.env" +RELAY_PID_DIR="${PHLEX_RELAY_STATE_DIR}/pids" +mkdir -p "$RELAY_PID_DIR" + +# Default exports (updated if relays are configured) +export PHLEX_HOST_GATEWAY="host.docker.internal" +export PHLEX_HOST_RELAY_FILE="$RELAY_MAP_JSON" +export PHLEX_HOST_RELAYS_ENV="$RELAY_MAP_ENV" +export PHLEX_PODMAN_SOCKET_SOURCE="podman-machine" +export PHLEX_RELAY_STATE_DIR + +# --- Helper functions --- + clone_if_absent() { local repo=$1 local dest="${PARENT_DIR}/${repo}" @@ -30,8 +136,6 @@ clone_if_absent() { return 0 fi - # Create the directory so the bind mount has a valid source even if the - # clone fails. mkdir -p "${dest}" echo "Cloning Framework-R-D/${repo} into ${dest} ..." @@ -43,8 +147,6 @@ clone_if_absent() { echo "WARNING: unable to clone ${repo} to ${dest}; an empty directory was created" >&2 return 0 fi - # Remove any partial clone before retrying; recreate the empty directory - # so the bind mount source remains valid. rm -rf "${dest}" mkdir -p "${dest}" sleep 5 @@ -55,94 +157,826 @@ ensure_bind_dir() { mkdir -p "$@" } -# start_socat_relay LABEL KILL_PATTERN LISTEN_SPEC CONNECT_SPEC LOGFILE READY_TEST -# -# Start a background socat relay, waiting up to 2 s for it to become ready. -# -# LABEL - human-readable name for log messages -# KILL_PATTERN - argument to pkill -f to stop any existing relay -# LISTEN_SPEC - socat first address (the listening side) -# CONNECT_SPEC - socat second address (the connecting side) -# LOGFILE - path for socat stdout/stderr -# READY_TEST - shell command whose exit code indicates readiness -# -# Returns 0 if the relay is ready, 1 otherwise. +RELAY_ALLOWLIST="25114=25115,25300=25301" +PHLEX_RELAY_ALLOWED_RANGE="${PHLEX_RELAY_ALLOWED_RANGE:-10.88.0.0/16}" +READ_PID="" + +declare -a valid_source_ports=() +declare -a valid_dest_ports=() +SELF_TEST_BLOCKER_PID="" +SELF_TEST_LISTENER_25114_PID="" +SELF_TEST_LISTENER_25300_PID="" +SELF_TEST_ROGUE_PID="" +SELF_TEST_SAVED_HOST_PORTS="" +SELF_TEST_SAVED_HOST_PORTS_SET=0 +SELF_TEST_SAVED_TEST_PORTS="" +SELF_TEST_SAVED_TEST_PORTS_SET=0 + +trim_whitespace() { + printf '%s' "$1" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' +} + +is_darwin() { + [[ "$OSTYPE" == darwin* ]] +} + +relay_pid_file() { + printf '%s/%s\n' "$RELAY_PID_DIR" "$1" +} + +allowed_destination_port() { + case "$1" in + 25114) + printf '25115\n' + ;; + 25300) + printf '25301\n' + ;; + *) + return 1 + ;; + esac +} + +is_allowed_mapping() { + case "$1=$2" in + 25114=25115|25300=25301) + return 0 + ;; + *) + return 1 + ;; + esac +} + +numeric_listen_spec() { + local dest_port="$1" + + if is_darwin; then + printf '%s\n' "TCP-LISTEN:${dest_port},bind=127.0.0.1,fork,reuseaddr" + else + printf '%s\n' "TCP-LISTEN:${dest_port},fork,reuseaddr,range=${PHLEX_RELAY_ALLOWED_RANGE}" + fi +} + +numeric_connect_spec() { + printf '%s\n' "TCP:127.0.0.1:$1" +} + +loopback_listener_ready_darwin() { + local port="$1" + + command -v lsof >/dev/null 2>&1 || return 1 + lsof -nP -iTCP:"$port" -sTCP:LISTEN -Fn 2>/dev/null | awk -v v4="n127.0.0.1:$port" -v v6="n[::1]:$port" ' + $0 == v4 || $0 == v6 { found = 1 } + END { exit !found } + ' +} + +relay_listener_ready_darwin() { + local port="$1" + + command -v lsof >/dev/null 2>&1 || return 1 + lsof -nP -iTCP:"$port" -sTCP:LISTEN -Fn 2>/dev/null | awk -v v4="n127.0.0.1:$port" ' + $0 == v4 { found = 1 } + END { exit !found } + ' +} + +loopback_listener_ready_linux() { + local port="$1" + + command -v ss >/dev/null 2>&1 || return 1 + ss -H -ltn 2>/dev/null | awk -v v4="127.0.0.1:$port" -v v6="[::1]:$port" ' + $4 == v4 || $4 == v6 { found = 1 } + END { exit !found } + ' +} + +relay_listener_ready_linux() { + local pid="${2:-}" + local port="$1" + + command -v ss >/dev/null 2>&1 || return 1 + ss -H -ltn 2>/dev/null | awk -v v4="0.0.0.0:$port" -v v6="[::]:$port" -v star="*:$port" ' + $4 == v4 || $4 == v6 || $4 == star { found = 1 } + END { exit !found } + ' || return 1 + + if [ -n "$pid" ]; then + linux_relay_process_has_allowed_range "$pid" + fi +} + +linux_relay_process_has_allowed_range() { + local cmdline + local pid="$1" + + cmdline=$(ps -p "$pid" -o command= 2>/dev/null) || return 1 + [[ "$cmdline" == *"range=${PHLEX_RELAY_ALLOWED_RANGE}"* ]] +} + +detect_source_listener() { + if is_darwin; then + loopback_listener_ready_darwin "$1" + else + loopback_listener_ready_linux "$1" + fi +} + +detect_numeric_relay_listener() { + if is_darwin; then + relay_listener_ready_darwin "$1" + else + relay_listener_ready_linux "$1" + fi +} + +aux_tcp_listener_ready() { + local port="$1" + + if is_darwin; then + command -v lsof >/dev/null 2>&1 || return 1 + lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1 + else + command -v ss >/dev/null 2>&1 || return 1 + ss -H -ltn 2>/dev/null | awk -v v4="0.0.0.0:$port" -v v6="[::]:$port" -v star="*:$port" ' + $4 == v4 || $4 == v6 || $4 == star { found = 1 } + END { exit !found } + ' + fi +} + +numeric_pid_has_listener_darwin() { + local pid="$1" + local dest_port="$2" + + command -v lsof >/dev/null 2>&1 || return 1 + lsof -a -p "$pid" -nP -iTCP:"$dest_port" -sTCP:LISTEN -Fn 2>/dev/null | awk -v v4="n127.0.0.1:$dest_port" ' + $0 == v4 { found = 1 } + END { exit !found } + ' +} + +numeric_pid_has_listener_linux() { + local pid="$1" + local dest_port="$2" + + command -v ss >/dev/null 2>&1 || return 1 + ss -H -ltnp 2>/dev/null | awk -v pid_fragment="pid=$pid," -v v4="0.0.0.0:$dest_port" -v v6="[::]:$dest_port" -v star="*:$dest_port" ' + ($4 == v4 || $4 == v6 || $4 == star) && index($0, pid_fragment) { found = 1 } + END { exit !found } + ' || return 1 + + relay_listener_ready_linux "$dest_port" "$pid" +} + +numeric_pid_has_listener() { + if is_darwin; then + numeric_pid_has_listener_darwin "$1" "$2" + else + numeric_pid_has_listener_linux "$1" "$2" + fi +} + +read_numeric_pidfile() { + local pid_contents + local pidfile="$1" + + READ_PID="" + + [ -f "$pidfile" ] || return 1 + pid_contents=$(cat "$pidfile" 2>/dev/null) || return 1 + pid_contents=$(trim_whitespace "$pid_contents") + [[ "$pid_contents" =~ ^[0-9]+$ ]] || return 1 + + READ_PID="$pid_contents" + return 0 +} + +wait_for_pid_exit() { + local attempts=0 + local pid="$1" + + while [ "$attempts" -lt 40 ]; do + if ! kill -0 "$pid" 2>/dev/null; then + return 0 + fi + sleep 0.1 + attempts=$((attempts + 1)) + done + + return 1 +} + +terminate_pid_gracefully() { + local pid="$1" + + kill "$pid" 2>/dev/null || return 1 + wait_for_pid_exit "$pid" +} + +numeric_commandline_matches() { + local cmdline + local connect_spec + local dest_port="$3" + local listen_spec + local pid="$1" + local source_port="$2" + + cmdline=$(ps -p "$pid" -o command= 2>/dev/null) || return 1 + listen_spec=$(numeric_listen_spec "$dest_port") + connect_spec=$(numeric_connect_spec "$source_port") + + case "$cmdline" in + *socat*"$listen_spec"*"$connect_spec"*) + return 0 + ;; + *socat*"$connect_spec"*"$listen_spec"*) + return 0 + ;; + *) + return 1 + ;; + esac +} + +numeric_relay_is_owned() { + local dest_port="$3" + local pid="$1" + local source_port="$2" + + kill -0 "$pid" 2>/dev/null || return 1 + numeric_commandline_matches "$pid" "$source_port" "$dest_port" || return 1 + numeric_pid_has_listener "$pid" "$dest_port" || return 1 + + return 0 +} + +resolve_existing_numeric_relay_state() { + local dest_port="$2" + local pid + local pidfile + local source_port="$1" + + pidfile=$(relay_pid_file "$source_port") + + if [ ! -f "$pidfile" ]; then + return 0 + fi + + if ! read_numeric_pidfile "$pidfile"; then + rm -f "$pidfile" + return 0 + fi + pid="$READ_PID" + + if ! kill -0 "$pid" 2>/dev/null; then + rm -f "$pidfile" + return 0 + fi + + if numeric_relay_is_owned "$pid" "$source_port" "$dest_port"; then + return 10 + fi + + if numeric_commandline_matches "$pid" "$source_port" "$dest_port"; then + if terminate_pid_gracefully "$pid"; then + rm -f "$pidfile" + return 0 + fi + + echo "WARNING: relay PID $pid for ${source_port} -> ${dest_port} did not stop cleanly" >&2 + return 11 + fi + + # PID reuse must not block startup or grant ownership of an unrelated process. + echo "WARNING: relay PID $pid in $pidfile is not a matching relay; discarding stale PID file" >&2 + rm -f "$pidfile" + return $? +} + +wait_for_source_listener_ready() { + local attempts=0 + local port="$1" + + while [ "$attempts" -lt 40 ]; do + if detect_source_listener "$port"; then + return 0 + fi + sleep 0.1 + attempts=$((attempts + 1)) + done + + return 1 +} + +aux_unix_socket_ready() { + local value="$1" + + [ -S "$value" ] || return 1 + python3 - "$value" <<'PY' +import socket +import sys + +sock = socket.socket(socket.AF_UNIX) +sock.settimeout(0.5) +try: + sock.connect(sys.argv[1]) +finally: + sock.close() +PY +} + +aux_endpoint_ready_once() { + local kind="$1" + local value="$2" + + case "$kind" in + unix-socket) + aux_unix_socket_ready "$value" + ;; + tcp-port) + aux_tcp_listener_ready "$value" + ;; + *) + return 1 + ;; + esac +} + +wait_for_aux_endpoint_ready() { + local attempts=0 + local kind="$1" + local value="$2" + + while [ "$attempts" -lt 40 ]; do + aux_endpoint_ready_once "$kind" "$value" && return 0 + sleep 0.1 + attempts=$((attempts + 1)) + done + + return 1 +} + +# start_socat_relay LABEL LISTEN_SPEC CONNECT_SPEC LOGFILE READY_KIND READY_VALUE start_socat_relay() { + local connect_spec="$3" local label="$1" - local kill_pattern="$2" - local listen_spec="$3" - local connect_spec="$4" - local logfile="$5" - local ready_test="$6" + local listen_spec="$2" + local logfile="$4" + local pid + local ready_kind="$5" + local ready_value="$6" if ! command -v socat >/dev/null 2>&1; then echo "WARNING: socat not found; cannot start ${label} relay" >&2 return 1 fi - # Stop any pre-existing relay process. - pkill -f "${kill_pattern}" 2>/dev/null || true - sleep 0.2 + if aux_endpoint_ready_once "$ready_kind" "$ready_value"; then + echo "${label} relay already ready" + return 0 + fi echo "Starting ${label} relay: ${listen_spec} -> ${connect_spec} ..." - nohup setsid socat "${listen_spec}" "${connect_spec}" > "${logfile}" 2>&1 & + nohup socat "$listen_spec" "$connect_spec" > "$logfile" 2>&1 & + pid=$! + + if wait_for_aux_endpoint_ready "$ready_kind" "$ready_value"; then + echo "${label} relay ready" + return 0 + fi + + if kill -0 "$pid" 2>/dev/null; then + terminate_pid_gracefully "$pid" >/dev/null 2>&1 || true + fi + + echo "WARNING: ${label} relay did not become ready in time" >&2 + return 1 +} + +parse_and_validate_relays() { + local dest_port + local duplicate_found + local invalid=0 + local pair + local relay_input="${1:-}" + local seen_source + local source_port + local -a relay_pairs=() + local -a seen_sources=() + + valid_source_ports=() + valid_dest_ports=() + + if [ -z "$relay_input" ]; then + return 0 + fi + + IFS=',' read -r -a relay_pairs <<< "$relay_input" + + for pair in "${relay_pairs[@]}"; do + pair=$(trim_whitespace "$pair") + + if [ -z "$pair" ] || [[ ! "$pair" =~ ^[^=]+=[^=]+$ ]]; then + echo "ERROR: malformed relay mapping: '$pair' (expected SOURCE=DEST)" >&2 + invalid=1 + continue + fi + + source_port=$(trim_whitespace "${pair%%=*}") + dest_port=$(trim_whitespace "${pair#*=}") + + if [[ ! "$source_port" =~ ^[0-9]+$ ]] || [[ ! "$dest_port" =~ ^[0-9]+$ ]]; then + echo "ERROR: relay mapping ports must be numeric: '$pair'" >&2 + invalid=1 + continue + fi + + if (( source_port < 1024 || dest_port < 1024 || source_port > 65535 || dest_port > 65535 )); then + echo "ERROR: relay ports must be between 1024 and 65535: '$pair'" >&2 + invalid=1 + continue + fi + + if (( source_port == dest_port )); then + echo "ERROR: source and destination ports must be different: '$pair'" >&2 + invalid=1 + continue + fi + + duplicate_found=0 + if [ "${#seen_sources[@]}" -gt 0 ]; then + for seen_source in "${seen_sources[@]}"; do + if [ "$seen_source" = "$source_port" ]; then + duplicate_found=1 + break + fi + done + fi + if [ "$duplicate_found" -eq 1 ]; then + echo "ERROR: duplicate source port in relay mappings: '$source_port'" >&2 + invalid=1 + continue + fi + seen_sources+=("$source_port") - # Wait up to 2 s for the relay to become ready. + if ! is_allowed_mapping "$source_port" "$dest_port"; then + echo "ERROR: relay mapping is not in the allowlist: '$pair'" >&2 + invalid=1 + continue + fi + + if ! detect_source_listener "$source_port"; then + echo "ERROR: source port $source_port is not listening on loopback" >&2 + invalid=1 + continue + fi + + valid_source_ports+=("$source_port") + valid_dest_ports+=("$dest_port") + done + + if [ "$invalid" -ne 0 ]; then + valid_source_ports=() + valid_dest_ports=() + return 1 + fi + + return 0 +} + +write_relay_maps() { local i=0 - while [ "${i}" -lt 20 ]; do - eval "${ready_test}" 2>/dev/null && break + local pair_count=0 + local sorted_pairs="" + + if [ -n "${valid_source_ports[*]:-}" ]; then + pair_count=${#valid_source_ports[@]} + fi + + if [ "$pair_count" -eq 0 ]; then + printf '{}\n' > "$RELAY_MAP_JSON" + : > "$RELAY_MAP_ENV" + return 0 + fi + + sorted_pairs=$(for ((i = 0; i < pair_count; i++)); do + printf '%s %s\n' "${valid_source_ports[$i]}" "${valid_dest_ports[$i]}" + done | LC_ALL=C sort -k1,1n) || return 1 + + { + local dest_port + local first=1 + local source_port + + printf '{' + while IFS=' ' read -r source_port dest_port; do + [ -n "$source_port" ] || continue + if [ "$first" -eq 1 ]; then + first=0 + else + printf ',' + fi + printf '"%s":"%s"' "$source_port" "$dest_port" + done < "$RELAY_MAP_JSON" + + { + local dest_port + local source_port + + while IFS=' ' read -r source_port dest_port; do + [ -n "$source_port" ] || continue + printf 'PHLEX_HOST_RELAY_%s=%s\n' "$source_port" "$dest_port" + done < "$RELAY_MAP_ENV" +} + +start_numeric_relay() { + local attempts=0 + local connect_spec + local dest_port="$2" + local listen_spec + local logfile + local pid + local pidfile + local source_port="$1" + local state_status=0 + + if ! command -v socat >/dev/null 2>&1; then + echo "ERROR: socat not found; cannot start host relay ${source_port}=${dest_port}" >&2 + return 1 + fi + + pidfile=$(relay_pid_file "$source_port") + resolve_existing_numeric_relay_state "$source_port" "$dest_port" + state_status=$? + case "$state_status" in + 0) + ;; + 10) + echo "Host relay $source_port -> $dest_port already ready" + return 0 + ;; + 11) + return 1 + ;; + *) + echo "WARNING: unexpected relay state status $state_status for ${source_port}=${dest_port}" >&2 + return 1 + ;; + esac + + if detect_numeric_relay_listener "$dest_port"; then + echo "WARNING: destination port $dest_port is already listening but not owned by this state" >&2 + return 1 + fi + + listen_spec=$(numeric_listen_spec "$dest_port") + connect_spec=$(numeric_connect_spec "$source_port") + logfile="${PHLEX_RELAY_STATE_DIR}/relay-${source_port}-${dest_port}.log" + mkdir -p "$RELAY_PID_DIR" + + echo "Starting host relay: 127.0.0.1:${source_port} -> ${dest_port}" + nohup socat "$listen_spec" "$connect_spec" > "$logfile" 2>&1 & + pid=$! + printf '%s\n' "$pid" > "$pidfile" + + while [ "$attempts" -lt 40 ]; do + if numeric_relay_is_owned "$pid" "$source_port" "$dest_port"; then + echo "Host relay $source_port -> $dest_port ready" + return 0 + fi + + if ! kill -0 "$pid" 2>/dev/null; then + break + fi + sleep 0.1 - i=$(( i + 1 )) + attempts=$((attempts + 1)) done - if eval "${ready_test}" 2>/dev/null; then - echo "${label} relay ready" - return 0 + echo "WARNING: host relay $source_port -> $dest_port did not become ready in time" >&2 + + if kill -0 "$pid" 2>/dev/null && numeric_commandline_matches "$pid" "$source_port" "$dest_port"; then + if ! terminate_pid_gracefully "$pid"; then + echo "WARNING: failed relay PID $pid for ${source_port}=${dest_port} did not stop cleanly" >&2 + return 1 + fi + fi + + rm -f "$pidfile" + return 1 +} + +apply_requested_relays() { + local i=0 + local overall_status=0 + local relay_input="$1" + local requested_count=0 + local -a ready_dest_ports=() + local -a ready_source_ports=() + + if ! parse_and_validate_relays "$relay_input"; then + write_relay_maps + return 1 + fi + + if [ -n "${valid_source_ports[*]:-}" ]; then + requested_count=${#valid_source_ports[@]} + fi + + for ((i = 0; i < requested_count; i++)); do + if start_numeric_relay "${valid_source_ports[$i]}" "${valid_dest_ports[$i]}"; then + ready_source_ports+=("${valid_source_ports[$i]}") + ready_dest_ports+=("${valid_dest_ports[$i]}") + else + overall_status=1 + fi + done + + if [ "${#ready_source_ports[@]}" -gt 0 ]; then + valid_source_ports=("${ready_source_ports[@]}") + valid_dest_ports=("${ready_dest_ports[@]}") else - echo "WARNING: ${label} relay did not become ready in time" >&2 + valid_source_ports=() + valid_dest_ports=() + fi + + write_relay_maps + return "$overall_status" +} + +mode_default() { + echo "Running in default mode with PHLEX_HOST_RELAY_PORTS" + + if [ -n "${PHLEX_HOST_RELAY_PORTS:-}" ]; then + apply_requested_relays "$PHLEX_HOST_RELAY_PORTS" + return $? + fi + + echo "No PHLEX_HOST_RELAY_PORTS configured; using defaults" + valid_source_ports=() + valid_dest_ports=() + write_relay_maps + return 0 +} + +mode_start_test_services() { + local test_ports="${PHLEX_TEST_RELAY_PORTS:-$RELAY_ALLOWLIST}" + + echo "Starting test relay services" + apply_requested_relays "$test_ports" +} + +cleanup_owned_numeric_relay() { + local dest_port + local pid + local pidfile + local source_port="$1" + + pidfile=$(relay_pid_file "$source_port") + [ -f "$pidfile" ] || return 0 + + dest_port=$(allowed_destination_port "$source_port") || { + echo "WARNING: relay PID file for unallowlisted source $source_port left untouched" >&2 + return 1 + } + + if ! read_numeric_pidfile "$pidfile"; then + rm -f "$pidfile" + return 0 + fi + pid="$READ_PID" + + if ! kill -0 "$pid" 2>/dev/null; then + rm -f "$pidfile" + return 0 + fi + + if ! numeric_commandline_matches "$pid" "$source_port" "$dest_port"; then + echo "WARNING: relay PID $pid in $pidfile is not a matching relay; discarding stale PID file" >&2 + rm -f "$pidfile" + return $? + fi + + if ! numeric_relay_is_owned "$pid" "$source_port" "$dest_port"; then + echo "WARNING: relay PID $pid for ${source_port} -> ${dest_port} was not proven owned; leaving it untouched" >&2 return 1 fi + + if ! terminate_pid_gracefully "$pid"; then + echo "WARNING: relay PID $pid for ${source_port} -> ${dest_port} did not stop cleanly" >&2 + return 1 + fi + + rm -f "$pidfile" + return 0 } +mode_cleanup_owned() { + local cleanup_failed=0 + local pidfile + local source_port + + echo "Cleaning up owned relay processes" + + if [ -d "$RELAY_PID_DIR" ]; then + for pidfile in "$RELAY_PID_DIR"/*; do + [ -e "$pidfile" ] || continue + source_port=$(basename "$pidfile") + case "$source_port" in + '' | *[!0-9]*) + continue + ;; + esac + if ! cleanup_owned_numeric_relay "$source_port"; then + cleanup_failed=1 + fi + done + fi + + valid_source_ports=() + valid_dest_ports=() + write_relay_maps + + if [ "$cleanup_failed" -ne 0 ]; then + echo "WARNING: one or more relay processes were not proven owned and were left untouched" >&2 + return 1 + fi + + echo "Cleanup completed" + return 0 +} + +mode_self_test() { + # Production startup and cleanup do not need fixture listeners or assertions. + if ! source "${SCRIPT_DIR}/ensure-repos-self-test.sh"; then + echo "ERROR: failed to source self-test helper" >&2 + return 1 + fi + mode_self_test_loaded +} + +# --- Main execution --- + +# Fixture and cleanup modes must not clone repositories, create bind mounts, +# inspect Podman, or create production relay state. +case "$MODE" in + self-test) + mode_self_test + exit $? + ;; + cleanup-owned) + mode_cleanup_owned + exit $? + ;; + start-test-services) + mode_start_test_services + exit $? + ;; +esac + clone_if_absent phlex-design clone_if_absent phlex-examples clone_if_absent phlex-coding-guidelines clone_if_absent phlex-spack-recipes # --- Podman Socket Proxy for Nested Containers (act) --- -# -# Rootless Podman nested volume mounts (like act's Docker SDK mounting the -# Podman socket) require that the source and target paths match across the -# host/container boundary. We use 'socat' on the host to provide a proxy socket -# at a stable, matching path in the user's home directory. - +# macOS sockets cannot cross the VM's shared filesystem. configure-host.py +# selects the VM-side socket and pasta reaches host loopback services directly. +if ! is_darwin; then +AUX_RELAY_LOG_DIR="${HOME}/.phlex-devcontainer-tmp/relay-logs" USER_ID=$(id -u) PODMAN_REAL_SOCKET="${XDG_RUNTIME_DIR:-/run/user/${USER_ID}}/podman/podman.sock" PROXY_DIR="${HOME}/.podman-proxy" PROXY_SOCKET="${PROXY_DIR}/podman.sock" +ensure_bind_dir -m 0700 "${AUX_RELAY_LOG_DIR}" ensure_bind_dir -m 0700 "${PROXY_DIR}" if [ -S "${PODMAN_REAL_SOCKET}" ]; then start_socat_relay \ "Podman socket" \ - "socat UNIX-LISTEN:${PROXY_SOCKET}" \ "UNIX-LISTEN:${PROXY_SOCKET},fork,reuseaddr,unlink-early" \ "UNIX-CONNECT:${PODMAN_REAL_SOCKET}" \ - "/tmp/socat-podman.log" \ - "[ -S '${PROXY_SOCKET}' ]" || true + "${AUX_RELAY_LOG_DIR}/socat-podman.log" \ + "unix-socket" \ + "${PROXY_SOCKET}" || true else echo "WARNING: Podman socket not found at ${PODMAN_REAL_SOCKET}" >&2 echo " To use 'act' inside the devcontainer, enable the Podman socket:" >&2 echo " systemctl --user enable --now podman.socket" >&2 fi -# Ensure the bind-mount source always exists so 'podman run' never fails with -# "no such file or directory", even when socat is unavailable or the real -# Podman socket is absent. if [ ! -S "${PROXY_SOCKET}" ]; then - # Create a dummy socket so the mount source is valid. The container will - # start successfully; nested container support (act) simply won't work. python3 -c " import socket, os s = socket.socket(socket.AF_UNIX) @@ -151,45 +985,23 @@ s.bind('${PROXY_SOCKET}') fi # --- Headroom Proxy Relays for Devcontainer --- -# -# The headroom proxy was split into TWO SSH-tunnelled ports, each bound only to -# 127.0.0.1 on this host: -# HEADROOM_AZURE_PORT (9797) : optimized proxy, Kilo provider fnal-azure -# HEADROOM_OW_PORT (9798) : passthrough proxy, Kilo provider fnal-ow -# Rootless Podman uses pasta for container networking, so containers reach the -# host via host.docker.internal (169.254.1.2) rather than via a bridge -# interface. However, pasta maps host.docker.internal to the host's loopback -# only for ports that are actually listening on all interfaces -- headroom's -# ports are bound to 127.0.0.1 only and are therefore unreachable. -# -# We relay EACH headroom port onto a different port on 0.0.0.0 so that -# containers can reach them via host.docker.internal:. A different -# port is required because 0.0.0.0: would conflict with the existing -# 127.0.0.1: listener. KILO_CONFIG_CONTENT_DOCKER (built in the shell rc) -# rewrites both loopback ports to their host.docker.internal relay ports and is -# wired into /root/.bashrc by post-create.sh. - HEADROOM_AZURE_PORT="${HEADROOM_AZURE_PORT:-9797}" HEADROOM_OW_PORT="${HEADROOM_OW_PORT:-9798}" -# relay_headroom_port ROLE PROXY_PORT -# Relay 127.0.0.1:PROXY_PORT to 0.0.0.0:(PROXY_PORT+10000) if the proxy is -# listening; otherwise warn and skip. Each role gets an independent relay -# and log file so the two never collide. relay_headroom_port() { local role="$1" local proxy_port="$2" local relay_port=$(( proxy_port + 10000 )) local local_addr="127.0.0.1:${proxy_port}" - if ss -tlnp 2>/dev/null | grep -q "127.0.0.1:${proxy_port}"; then + if detect_source_listener "${proxy_port}"; then start_socat_relay \ "Headroom ${role} proxy" \ - "socat TCP-LISTEN:${relay_port}" \ "TCP-LISTEN:${relay_port},fork,reuseaddr" \ "TCP:${local_addr}" \ - "/tmp/socat-headroom-${role}.log" \ - "ss -tlnp 2>/dev/null | grep -q ':${relay_port} '" || true + "${AUX_RELAY_LOG_DIR}/socat-headroom-${role}.log" \ + "tcp-port" \ + "${relay_port}" || true else echo "WARNING: headroom ${role} proxy not detected at ${local_addr}; skipping relay" >&2 echo " Ensure the SSH tunnel is active (headroom-${role} running on your laptop)" >&2 @@ -198,6 +1010,7 @@ relay_headroom_port() { relay_headroom_port azure "${HEADROOM_AZURE_PORT}" relay_headroom_port ow "${HEADROOM_OW_PORT}" +fi # Ensure remaining source bind mount points exist. ensure_bind_dir "$HOME/.config/"{gh,kilo} @@ -206,4 +1019,14 @@ ensure_bind_dir -m 0700 "$HOME/.vscode-remote-user-data" ensure_bind_dir -m 0700 "$HOME/.local/share/kilo" ensure_bind_dir -m 0700 "$HOME/.phlex-devcontainer-tmp" +# Execute the normal host setup after the legacy repository and Podman setup. +if ! mode_default; then + echo "ERROR: relay configuration rejected; no relay map was admitted" >&2 + exit 1 +fi + +# The initializer is a child process: exporting these settings here cannot +# affect the later Compose process launched by VS Code. +python3 "${SCRIPT_DIR}/configure-host.py" || exit 1 + echo "SUCCESS: .devcontainer/ensure-repos.sh completed successfully" diff --git a/.devcontainer/kilo-env-self-test.sh b/.devcontainer/kilo-env-self-test.sh new file mode 100644 index 000000000..eae2553ab --- /dev/null +++ b/.devcontainer/kilo-env-self-test.sh @@ -0,0 +1,933 @@ +#!/usr/bin/env bash +# Test-only helper for kilo-env.sh - contains assertion and test functions +# This file is sourced only when kilo_env_run_self_tests is invoked +# Do not execute directly; returns error if executed directly (not sourced) + +# If executed directly (BASH_SOURCE[0] == $0), return error +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + printf 'kilo-env-self-test: this file is a test helper and must not be executed directly (only sourced)\n' >&2 + exit 1 +fi + +# === Test-only assertion functions === + +kilo_env_sha256() { + kilo_env_require_args 1 "${FUNCNAME[0]}" "$@" || return 2 + local path + local sum_line + + path="$1" + sum_line="" + + if command -v shasum >/dev/null 2>&1 && \ + sum_line=$(shasum -a 256 "$path" 2>/dev/null); then + printf '%s\n' "${sum_line%% *}" + return 0 + fi + + if command -v sha256sum >/dev/null 2>&1 && \ + sum_line=$(sha256sum "$path" 2>/dev/null); then + printf '%s\n' "${sum_line%% *}" + return 0 + fi + + return 1 +} + +kilo_env_assert_equal() { + kilo_env_require_args 3 "${FUNCNAME[0]}" "$@" || return 2 + local actual + local expected + local label + + actual="$1" + expected="$2" + label="$3" + + if [ "$actual" = "$expected" ]; then + printf 'PASS: %s\n' "$label" + return 0 + fi + + printf 'FAIL: %s\n' "$label" >&2 + printf ' expected: %s\n' "$expected" >&2 + printf ' actual: %s\n' "$actual" >&2 + return 1 +} + +kilo_env_assert_file_contains() { + kilo_env_require_args 3 "${FUNCNAME[0]}" "$@" || return 2 + local path + local needle + local label + + path="$1" + needle="$2" + label="$3" + + if grep -F "$needle" "$path" >/dev/null 2>&1; then + printf 'PASS: %s\n' "$label" + return 0 + fi + + printf 'FAIL: %s\n' "$label" >&2 + if [ -f "$path" ]; then + printf ' file: %s\n' "$path" >&2 + fi + return 1 +} + +kilo_env_assert_file_empty() { + kilo_env_require_args 2 "${FUNCNAME[0]}" "$@" || return 2 + local path + local label + + path="$1" + label="$2" + + if [ ! -s "$path" ]; then + printf 'PASS: %s\n' "$label" + return 0 + fi + + printf 'FAIL: %s\n' "$label" >&2 + return 1 +} + +kilo_env_assert_export_value() { + kilo_env_require_args 3 "${FUNCNAME[0]}" "$@" || return 2 + local variable_name + local expected + local label + local actual + + variable_name="$1" + expected="$2" + label="$3" + actual=$(printenv "$variable_name" 2>/dev/null || true) + + kilo_env_assert_equal "$actual" "$expected" "$label" +} + +kilo_env_assert_export_absent() { + kilo_env_require_args 2 "${FUNCNAME[0]}" "$@" || return 2 + local variable_name + local label + + variable_name="$1" + label="$2" + + if printenv "$variable_name" >/dev/null 2>&1; then + printf 'FAIL: %s\n' "$label" >&2 + return 1 + fi + + printf 'PASS: %s\n' "$label" + return 0 +} + +kilo_env_assert_hash_unchanged() { + kilo_env_require_args 3 "${FUNCNAME[0]}" "$@" || return 2 + local path + local before_hash + local label + local after_hash + + path="$1" + before_hash="$2" + label="$3" + after_hash="" + + if ! after_hash=$(kilo_env_sha256 "$path"); then + printf 'FAIL: %s\n' "$label" >&2 + return 1 + fi + + kilo_env_assert_equal "$after_hash" "$before_hash" "$label" +} + +kilo_env_extract_output_value() { + kilo_env_require_args 2 "${FUNCNAME[0]}" "$@" || return 2 + local content + local key + + content="$1" + key="$2" + + printf '%s\n' "$content" | awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1); exit}' +} + +kilo_env_create_self_test_root() { + local nonce + local scratch_root + + nonce="" + scratch_root="" + + if ! nonce=$(python3 - <<'PY' +import os +import time + +print(f"{os.getpid()}-{time.time_ns()}") +PY + ); then + return 1 + fi + + if ! scratch_root=$(mktemp -d "${TMPDIR:-/tmp}/phlex-kilo-self-test.${nonce}.XXXXXX"); then + return 1 + fi + + printf '%s\n' "$scratch_root" +} + +kilo_env_cleanup_self_test_root() { + kilo_env_require_args 1 "${FUNCNAME[0]}" "$@" || return 2 + local scratch_root + + scratch_root="$1" + case "$scratch_root" in + "${TMPDIR:-/tmp}"/phlex-kilo-self-test.*) + rm -rf "$scratch_root" + ;; + esac +} + +kilo_env_seed_stale_exports() { + export PHLEX_HOST_RELAY_25114="stale" + export PHLEX_HOST_RELAY_25300="stale" + export KILO_CONFIG_CONTENT="stale" +} + +kilo_env_run_self_tests_impl() { + kilo_env_require_args 2 "${FUNCNAME[0]}" "$@" || return 2 + local fixture_root="$1" + local scratch_root="$2" + local fixture_map="${fixture_root}/relay-map.json" + local fixture_env="${fixture_root}/relay-map.env" + local fixture_artifact="${fixture_root}/artifact-sha256.txt" + local tests_failed=0 + + printf '=== Kilo config rewrite self-test ===\n' + printf 'INFO: native mount/runtime integration unavailable; fixture-only consumer logic validated.\n' + + if [ ! -f "$fixture_map" ] || [ ! -f "$fixture_env" ] || [ ! -f "$fixture_artifact" ]; then + kilo_env_print_error "PHLEX_RELAY_FIXTURE_DIR must contain relay-map.json, relay-map.env, and artifact-sha256.txt" + return 1 + fi + + local fixture_map_before fixture_env_before fixture_artifact_before + if ! fixture_map_before=$(kilo_env_sha256 "$fixture_map"); then + kilo_env_print_error "unable to hash relay fixture map" + return 1 + fi + if ! fixture_env_before=$(kilo_env_sha256 "$fixture_env"); then + kilo_env_print_error "unable to hash relay fixture env" + return 1 + fi + if ! fixture_artifact_before=$(kilo_env_sha256 "$fixture_artifact"); then + kilo_env_print_error "unable to hash relay fixture artifact manifest" + return 1 + fi + + local recorded_map_hash recorded_env_hash + recorded_map_hash=$(awk '/relay-map\.json$/ {print $1; exit}' "$fixture_artifact") + recorded_env_hash=$(awk '/relay-map\.env$/ {print $1; exit}' "$fixture_artifact") + + if ! kilo_env_assert_equal "$fixture_map_before" "$recorded_map_hash" \ + "carrier relay-map.json SHA-256 matches artifact receipt"; then + tests_failed=1 + fi + if ! kilo_env_assert_equal "$fixture_env_before" "$recorded_env_hash" \ + "carrier relay-map.env SHA-256 matches artifact receipt"; then + tests_failed=1 + fi + + printf 'INFO: carrier relay-map.json sha256=%s\n' "$fixture_map_before" + printf 'INFO: carrier relay-map.env sha256=%s\n' "$fixture_env_before" + + local expected_json_round_trip expected_jsonc_round_trip + expected_json_round_trip=$(cat <<'EOF' +{"provider":{"anthropic":{"options":{"baseURL":"https://host.docker.internal:25115/v1?source=a#frag"}},"openai":{"options":{"baseURL":"http://host.docker.internal:25301/api/v1"}},"ipv6":{"options":{"baseURL":"https://host.docker.internal:25115/chat?q=1#anchor"}},"nonloopback":{"options":{"baseURL":"https://api.example.com:25114/v1"}},"unmapped":{"options":{"baseURL":"http://127.0.0.1:29999/v1"}},"stdio":{"transport":{"type":"stdio","command":"kilo"}},"unix":{"options":{"baseURL":"unix:///var/run/kilo.sock"}},"nested":{"array":["https://host.docker.internal:25115/inside","http://localhost:9999/stay"]}},"other":"keep"} +EOF +) + expected_jsonc_round_trip=$(cat <<'EOF' +{"provider":{"commented":{"options":{"baseURL":"https://host.docker.internal:25115/v2"}},"literal":{"url":"http://host.docker.internal:25301/path//segment?x=1#frag","text":"/* not a comment */ // still text"}},"array":["https://host.docker.internal:25115/array"]} +EOF +) + + kilo_env_seed_stale_exports + if ! kilo_env_apply_root "$fixture_root"; then + printf 'FAIL: exact carrier root validation\n' >&2 + tests_failed=1 + else + printf 'PASS: exact carrier root validation\n' + fi + if ! kilo_env_assert_export_value "PHLEX_HOST_RELAY_25114" "25115" \ + "exact carrier exports PHLEX_HOST_RELAY_25114"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_value "PHLEX_HOST_RELAY_25300" "25301" \ + "exact carrier exports PHLEX_HOST_RELAY_25300"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "KILO_CONFIG_CONTENT" \ + "exact carrier root without config leaves KILO_CONFIG_CONTENT absent"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "KILO_CONFIG_PATH" \ + "exact carrier root without config leaves KILO_CONFIG_PATH absent"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_value "PHLEX_HOST_RELAY_FILE" "$fixture_map" \ + "exact carrier root updates PHLEX_HOST_RELAY_FILE"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_value "PHLEX_HOST_RELAYS_ENV" "$fixture_env" \ + "exact carrier root updates PHLEX_HOST_RELAYS_ENV"; then + tests_failed=1 + fi + + local exact_config="${scratch_root}/exact-carrier-config.json" + cat > "$exact_config" <<'EOF' +{ + "provider": { + "anthropic": { + "options": { + "baseURL": "https://127.0.0.1:25114/v1?source=a#frag" + } + }, + "openai": { + "options": { + "baseURL": "http://localhost:25300/api/v1" + } + }, + "ipv6": { + "options": { + "baseURL": "https://[::1]:25114/chat?q=1#anchor" + } + }, + "nonloopback": { + "options": { + "baseURL": "https://api.example.com:25114/v1" + } + }, + "unmapped": { + "options": { + "baseURL": "http://127.0.0.1:29999/v1" + } + }, + "stdio": { + "transport": { + "type": "stdio", + "command": "kilo" + } + }, + "unix": { + "options": { + "baseURL": "unix:///var/run/kilo.sock" + } + }, + "nested": { + "array": [ + "https://127.0.0.1:25114/inside", + "http://localhost:9999/stay" + ] + } + }, + "other": "keep" +} +EOF + local exact_config_hash + exact_config_hash=$(kilo_env_sha256 "$exact_config") + + kilo_env_seed_stale_exports + kilo_env_unset_var_quietly "KILO_CONFIG_PATH" + if ! kilo_env_apply_paths "$fixture_map" "$fixture_env" "$exact_config"; then + printf 'FAIL: exact carrier round-trip rewrite\n' >&2 + tests_failed=1 + else + printf 'PASS: exact carrier round-trip rewrite\n' + fi + if ! kilo_env_assert_export_value "PHLEX_HOST_RELAY_25114" "25115" \ + "exact carrier round-trip keeps PHLEX_HOST_RELAY_25114"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_value "PHLEX_HOST_RELAY_25300" "25301" \ + "exact carrier round-trip keeps PHLEX_HOST_RELAY_25300"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_value "KILO_CONFIG_CONTENT" "$expected_json_round_trip" \ + "exact carrier round-trip rewrites 127.0.0.1 localhost and [::1] URLs"; then + tests_failed=1 + fi + if ! kilo_env_assert_hash_unchanged "$exact_config" "$exact_config_hash" \ + "exact carrier config hash unchanged"; then + tests_failed=1 + fi + + local json_root="${scratch_root}/json-root" + mkdir -p "$json_root" + cat > "${json_root}/relay-map.json" <<'EOF' +{"25114":"25115","25300":"25301"} +EOF + cat > "${json_root}/relay-map.env" <<'EOF' +PHLEX_HOST_RELAY_25114=25115 +PHLEX_HOST_RELAY_25300=25301 +EOF + local json_config="${json_root}/kilo.json" + cat > "$json_config" <<'EOF' +{ + "provider": { + "anthropic": { + "options": { + "baseURL": "https://127.0.0.1:25114/v1?source=a#frag" + } + }, + "openai": { + "options": { + "baseURL": "http://localhost:25300/api/v1" + } + }, + "ipv6": { + "options": { + "baseURL": "https://[::1]:25114/chat?q=1#anchor" + } + }, + "nonloopback": { + "options": { + "baseURL": "https://api.example.com:25114/v1" + } + }, + "unmapped": { + "options": { + "baseURL": "http://127.0.0.1:29999/v1" + } + }, + "stdio": { + "transport": { + "type": "stdio", + "command": "kilo" + } + }, + "unix": { + "options": { + "baseURL": "unix:///var/run/kilo.sock" + } + }, + "nested": { + "array": [ + "https://127.0.0.1:25114/inside", + "http://localhost:9999/stay" + ] + } + }, + "other": "keep" +} +EOF + cat > "${json_root}/kilo.jsonc" <<'EOF' +{ + "decoy": { + "baseURL": "http://localhost:1/should-not-be-selected" + } +} +EOF + local json_config_hash + json_config_hash=$(kilo_env_sha256 "$json_config") + local json_decoy_hash + json_decoy_hash=$(kilo_env_sha256 "${json_root}/kilo.jsonc") + + kilo_env_seed_stale_exports + if ! kilo_env_apply_root "$json_root"; then + printf 'FAIL: JSON root rewrite\n' >&2 + tests_failed=1 + else + printf 'PASS: JSON root rewrite\n' + fi + if ! kilo_env_assert_export_value "KILO_CONFIG_PATH" "${json_root}/kilo.json" \ + "JSON root prefers kilo.json over kilo.jsonc"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_value "KILO_CONFIG_CONTENT" "$expected_json_round_trip" \ + "JSON root exports rewritten KILO_CONFIG_CONTENT"; then + tests_failed=1 + fi + if ! kilo_env_assert_hash_unchanged "$json_config" "$json_config_hash" \ + "JSON root kilo.json hash unchanged"; then + tests_failed=1 + fi + if ! kilo_env_assert_hash_unchanged "${json_root}/kilo.jsonc" "$json_decoy_hash" \ + "JSON root decoy kilo.jsonc hash unchanged"; then + tests_failed=1 + fi + + local jsonc_root="${scratch_root}/jsonc-root" + mkdir -p "$jsonc_root" + cat > "${jsonc_root}/relay-map.json" <<'EOF' +{"25114":"25115","25300":"25301"} +EOF + cat > "${jsonc_root}/relay-map.env" <<'EOF' +PHLEX_HOST_RELAY_25114=25115 +PHLEX_HOST_RELAY_25300=25301 +EOF + local jsonc_config="${jsonc_root}/kilo.jsonc" + cat > "$jsonc_config" <<'EOF' +{ + // provider comment + "provider": { + /* block comment */ + "commented": { + "options": { + "baseURL": "https://localhost:25114/v2", // trailing comment + }, + }, + "literal": { + "url": "http://localhost:25300/path//segment?x=1#frag", + "text": "/* not a comment */ // still text", + }, + }, + "array": [ + "https://[::1]:25114/array", + ], +} +EOF + local jsonc_config_hash + jsonc_config_hash=$(kilo_env_sha256 "$jsonc_config") + + kilo_env_seed_stale_exports + if ! kilo_env_apply_root "$jsonc_root"; then + printf 'FAIL: JSONC root rewrite\n' >&2 + tests_failed=1 + else + printf 'PASS: JSONC root rewrite\n' + fi + if ! kilo_env_assert_export_value "KILO_CONFIG_PATH" "${jsonc_root}/kilo.jsonc" \ + "JSONC root falls back to kilo.jsonc"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_value "KILO_CONFIG_CONTENT" "$expected_jsonc_round_trip" \ + "JSONC root strips comments/trailing commas and rewrites URLs"; then + tests_failed=1 + fi + if ! kilo_env_assert_hash_unchanged "$jsonc_config" "$jsonc_config_hash" \ + "JSONC root kilo.jsonc hash unchanged"; then + tests_failed=1 + fi + + local malformed_json_root="${scratch_root}/malformed-json-root" + mkdir -p "$malformed_json_root" + cat > "${malformed_json_root}/relay-map.json" <<'EOF' +{"25114":"25115","25300":"25301"} +EOF + cat > "${malformed_json_root}/relay-map.env" <<'EOF' +PHLEX_HOST_RELAY_25114=25115 +PHLEX_HOST_RELAY_25300=25301 +EOF + cat > "${malformed_json_root}/kilo.json" <<'EOF' +{ + "provider": { + "broken": { + "options": { + "baseURL": "https://127.0.0.1:25114/v1" + } + } +EOF + local malformed_json_hash + malformed_json_hash=$(kilo_env_sha256 "${malformed_json_root}/kilo.json") + local malformed_json_stderr="${malformed_json_root}/stderr.txt" + + kilo_env_seed_stale_exports + if kilo_env_apply_root "$malformed_json_root" 2>"$malformed_json_stderr"; then + printf 'FAIL: malformed JSON config must fail closed\n' >&2 + tests_failed=1 + else + printf 'PASS: malformed JSON config fails closed\n' + fi + if ! kilo_env_assert_file_contains "$malformed_json_stderr" \ + "invalid Kilo config JSON/JSONC" "malformed JSON reports a clear stderr error"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "PHLEX_HOST_RELAY_25114" \ + "malformed JSON clears PHLEX_HOST_RELAY_25114"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "PHLEX_HOST_RELAY_25300" \ + "malformed JSON clears PHLEX_HOST_RELAY_25300"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "KILO_CONFIG_CONTENT" \ + "malformed JSON clears KILO_CONFIG_CONTENT"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "KILO_CONFIG_PATH" \ + "malformed JSON clears KILO_CONFIG_PATH"; then + tests_failed=1 + fi + if ! kilo_env_assert_hash_unchanged "${malformed_json_root}/kilo.json" "$malformed_json_hash" \ + "malformed JSON input hash unchanged"; then + tests_failed=1 + fi + + local malformed_jsonc_root="${scratch_root}/malformed-jsonc-root" + mkdir -p "$malformed_jsonc_root" + cat > "${malformed_jsonc_root}/relay-map.json" <<'EOF' +{"25114":"25115","25300":"25301"} +EOF + cat > "${malformed_jsonc_root}/relay-map.env" <<'EOF' +PHLEX_HOST_RELAY_25114=25115 +PHLEX_HOST_RELAY_25300=25301 +EOF + cat > "${malformed_jsonc_root}/kilo.jsonc" <<'EOF' +{ + "provider": { + "broken": { + "options": { + "baseURL": "https://localhost:25114/v1" + } + } /* unterminated +EOF + local malformed_jsonc_hash + malformed_jsonc_hash=$(kilo_env_sha256 "${malformed_jsonc_root}/kilo.jsonc") + local malformed_jsonc_stderr="${malformed_jsonc_root}/stderr.txt" + + kilo_env_seed_stale_exports + if kilo_env_apply_root "$malformed_jsonc_root" 2>"$malformed_jsonc_stderr"; then + printf 'FAIL: malformed JSONC config must fail closed\n' >&2 + tests_failed=1 + else + printf 'PASS: malformed JSONC config fails closed\n' + fi + if ! kilo_env_assert_file_contains "$malformed_jsonc_stderr" \ + "unterminated block comment in Kilo config" \ + "malformed JSONC reports unterminated comment"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "PHLEX_HOST_RELAY_25114" \ + "malformed JSONC clears PHLEX_HOST_RELAY_25114"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "PHLEX_HOST_RELAY_25300" \ + "malformed JSONC clears PHLEX_HOST_RELAY_25300"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "KILO_CONFIG_CONTENT" \ + "malformed JSONC clears KILO_CONFIG_CONTENT"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "KILO_CONFIG_PATH" \ + "malformed JSONC clears KILO_CONFIG_PATH"; then + tests_failed=1 + fi + if ! kilo_env_assert_hash_unchanged "${malformed_jsonc_root}/kilo.jsonc" "$malformed_jsonc_hash" \ + "malformed JSONC input hash unchanged"; then + tests_failed=1 + fi + + local mismatched_root="${scratch_root}/mismatched-root" + mkdir -p "$mismatched_root" + cat > "${mismatched_root}/relay-map.json" <<'EOF' +{"25114":"25115","25300":"25301"} +EOF + cat > "${mismatched_root}/relay-map.env" <<'EOF' +PHLEX_HOST_RELAY_25114=25115 +EOF + local mismatched_stderr="${mismatched_root}/stderr.txt" + + kilo_env_seed_stale_exports + if kilo_env_apply_root "$mismatched_root" 2>"$mismatched_stderr"; then + printf 'FAIL: mismatched relay map/env must fail closed\n' >&2 + tests_failed=1 + else + printf 'PASS: mismatched relay map/env fails closed\n' + fi + if ! kilo_env_assert_file_contains "$mismatched_stderr" \ + "relay env must be sorted canonical" "mismatched relay map/env reports stderr error"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "PHLEX_HOST_RELAY_25114" \ + "mismatched relay map/env clears PHLEX_HOST_RELAY_25114"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "PHLEX_HOST_RELAY_25300" \ + "mismatched relay map/env clears PHLEX_HOST_RELAY_25300"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "KILO_CONFIG_CONTENT" \ + "mismatched relay map/env clears KILO_CONFIG_CONTENT"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "KILO_CONFIG_PATH" \ + "mismatched relay map/env clears KILO_CONFIG_PATH"; then + tests_failed=1 + fi + + local invalid_allowlist_root="${scratch_root}/invalid-allowlist-root" + mkdir -p "$invalid_allowlist_root" + cat > "${invalid_allowlist_root}/relay-map.json" <<'EOF' +{"9999":"10000"} +EOF + cat > "${invalid_allowlist_root}/relay-map.env" <<'EOF' +PHLEX_HOST_RELAY_9999=10000 +EOF + local invalid_allowlist_stderr="${invalid_allowlist_root}/stderr.txt" + + kilo_env_seed_stale_exports + if kilo_env_apply_root "$invalid_allowlist_root" 2>"$invalid_allowlist_stderr"; then + printf 'FAIL: invalid allowlist relay map must fail closed\n' >&2 + tests_failed=1 + else + printf 'PASS: invalid allowlist relay map fails closed\n' + fi + if ! kilo_env_assert_file_contains "$invalid_allowlist_stderr" \ + "is not allowlisted" "invalid allowlist relay map reports stderr error"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "PHLEX_HOST_RELAY_25114" \ + "invalid allowlist relay map leaves PHLEX_HOST_RELAY_25114 absent"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "KILO_CONFIG_CONTENT" \ + "invalid allowlist relay map leaves KILO_CONFIG_CONTENT absent"; then + tests_failed=1 + fi + + local duplicate_env_root="${scratch_root}/duplicate-env-root" + mkdir -p "$duplicate_env_root" + cat > "${duplicate_env_root}/relay-map.json" <<'EOF' +{"25114":"25115"} +EOF + cat > "${duplicate_env_root}/relay-map.env" <<'EOF' +PHLEX_HOST_RELAY_25114=25115 +PHLEX_HOST_RELAY_25114=25115 +EOF + local duplicate_env_stderr="${duplicate_env_root}/stderr.txt" + + kilo_env_seed_stale_exports + if kilo_env_apply_root "$duplicate_env_root" 2>"$duplicate_env_stderr"; then + printf 'FAIL: duplicate relay env must fail closed\n' >&2 + tests_failed=1 + else + printf 'PASS: duplicate relay env fails closed\n' + fi + if ! kilo_env_assert_file_contains "$duplicate_env_stderr" \ + "duplicate relay env entry" "duplicate relay env reports stderr error"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "PHLEX_HOST_RELAY_25114" \ + "duplicate relay env clears PHLEX_HOST_RELAY_25114"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "KILO_CONFIG_CONTENT" \ + "duplicate relay env clears KILO_CONFIG_CONTENT"; then + tests_failed=1 + fi + + local missing_root="${scratch_root}/missing-root" + local missing_root_stderr="${scratch_root}/missing-root.stderr" + + kilo_env_seed_stale_exports + if ! kilo_env_apply_root "$missing_root" 2>"$missing_root_stderr"; then + printf 'FAIL: absent relay root must fail closed without returning nonzero\n' >&2 + tests_failed=1 + else + printf 'PASS: absent relay root fails closed without returning nonzero\n' + fi + if ! kilo_env_assert_file_empty "$missing_root_stderr" \ + "absent relay root is quiet"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "PHLEX_HOST_RELAY_25114" \ + "absent relay root clears PHLEX_HOST_RELAY_25114"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "PHLEX_HOST_RELAY_25300" \ + "absent relay root clears PHLEX_HOST_RELAY_25300"; then + tests_failed=1 + fi + if ! kilo_env_assert_export_absent "KILO_CONFIG_CONTENT" \ + "absent relay root clears KILO_CONFIG_CONTENT"; then + tests_failed=1 + fi + + local script_path child_output source_safety_output_ready=0 + local isolated_root="${scratch_root}/isolated-fixed-root" + local fixed_root_replacements=0 + local line + script_path="${scratch_root}/kilo-env.sh" + + if ! mkdir -p "$isolated_root"; then + kilo_env_print_error "unable to create isolated source-safety relay root" + tests_failed=1 + elif ! { + while IFS= read -r line || [ -n "$line" ]; do + if [[ "$line" == KILO_ENV_FIXED_ROOT=* ]]; then + printf 'KILO_ENV_FIXED_ROOT=%q\n' "$isolated_root" + fixed_root_replacements=$((fixed_root_replacements + 1)) + else + printf '%s\n' "$line" + fi + done < "$KILO_ENV_SCRIPT_PATH" + } > "$script_path"; then + kilo_env_print_error \ + "unable to prepare isolated source-safety script" + tests_failed=1 + elif [ "$fixed_root_replacements" -ne 1 ]; then + kilo_env_print_error "source-safety script did not have exactly one fixed-root assignment" + tests_failed=1 + elif [ -e "${isolated_root}/relay-map.json" ] || [ -e "${isolated_root}/relay-map.env" ]; then + kilo_env_print_error \ + "source-safety test requires ${isolated_root} to contain neither relay-map.json nor relay-map.env" + tests_failed=1 + elif ! child_output=$( + PHLEX_RELAY_FIXTURE_DIR="$fixture_root" \ + bash --noprofile --norc -c ' +script_path=$1 +isolated_root=$2 +set -euf -o pipefail +before_flags=$- +before_opts=$(set +o) +export PHLEX_HOST_RELAY_25114=stale +export PHLEX_HOST_RELAY_25300=stale +export KILO_CONFIG_CONTENT=stale +export KILO_CONFIG_PATH=stale +. "$script_path" +status=$? +after_flags=$- +after_opts=$(set +o) +sha256_text() { + local input="$1" + local sum_line + if command -v shasum >/dev/null 2>&1 && \ + sum_line=$(printf %s "$input" | shasum -a 256 2>/dev/null); then + printf "%s\n" "$sum_line" + return 0 + fi + if command -v sha256sum >/dev/null 2>&1 && \ + sum_line=$(printf %s "$input" | sha256sum 2>/dev/null); then + printf "%s\n" "$sum_line" + return 0 + fi + return 1 +} +before_opts_sha=$(sha256_text "$before_opts") || exit 1 +before_opts_sha=${before_opts_sha%% *} +after_opts_sha=$(sha256_text "$after_opts") || exit 1 +after_opts_sha=${after_opts_sha%% *} +printf "%s\n" \ + "status=$status" \ + "before_flags=$before_flags" \ + "after_flags=$after_flags" \ + "before_opts_sha=$before_opts_sha" \ + "after_opts_sha=$after_opts_sha" \ + "relay25114=${PHLEX_HOST_RELAY_25114-unset}" \ + "relay25300=${PHLEX_HOST_RELAY_25300-unset}" \ + "config=${KILO_CONFIG_CONTENT-unset}" \ + "config_path=${KILO_CONFIG_PATH-unset}" \ + "relay_file=${PHLEX_HOST_RELAY_FILE-unset}" \ + "relays_env=${PHLEX_HOST_RELAYS_ENV-unset}" \ + "gateway=${PHLEX_HOST_GATEWAY-unset}" +' bash "$script_path" "$isolated_root" + ); then + printf 'FAIL: production source-safety child shell exited unexpectedly\n' >&2 + tests_failed=1 + else + printf 'PASS: production source-safety child shell stayed alive\n' + source_safety_output_ready=1 + fi + + if [ "$source_safety_output_ready" -eq 1 ]; then + if ! kilo_env_assert_equal "$(kilo_env_extract_output_value "$child_output" "status")" "0" \ + "production sourcing returns success"; then + tests_failed=1 + fi + if ! kilo_env_assert_equal "$(kilo_env_extract_output_value "$child_output" "before_flags")" \ + "$(kilo_env_extract_output_value "$child_output" "after_flags")" \ + "production sourcing preserves shell flags"; then + tests_failed=1 + fi + if ! kilo_env_assert_equal "$(kilo_env_extract_output_value "$child_output" "before_opts_sha")" \ + "$(kilo_env_extract_output_value "$child_output" "after_opts_sha")" \ + "production sourcing preserves shell options"; then + tests_failed=1 + fi + if ! kilo_env_assert_equal "$(kilo_env_extract_output_value "$child_output" "relay25114")" "unset" \ + "production sourcing clears stale PHLEX_HOST_RELAY_25114"; then + tests_failed=1 + fi + if ! kilo_env_assert_equal "$(kilo_env_extract_output_value "$child_output" "relay25300")" "unset" \ + "production sourcing clears stale PHLEX_HOST_RELAY_25300"; then + tests_failed=1 + fi + if ! kilo_env_assert_equal "$(kilo_env_extract_output_value "$child_output" "config")" "unset" \ + "production sourcing clears stale KILO_CONFIG_CONTENT"; then + tests_failed=1 + fi + if ! kilo_env_assert_equal "$(kilo_env_extract_output_value "$child_output" "config_path")" "unset" \ + "production sourcing clears stale KILO_CONFIG_PATH"; then + tests_failed=1 + fi + if ! kilo_env_assert_equal "$(kilo_env_extract_output_value "$child_output" "relay_file")" \ + "${isolated_root}/relay-map.json" \ + "source-safety copy uses the isolated relay-map.json path"; then + tests_failed=1 + fi + if ! kilo_env_assert_equal "$(kilo_env_extract_output_value "$child_output" "relays_env")" \ + "${isolated_root}/relay-map.env" \ + "source-safety copy uses the isolated relay-map.env path"; then + tests_failed=1 + fi + if ! kilo_env_assert_equal "$(kilo_env_extract_output_value "$child_output" "gateway")" \ + "$KILO_ENV_HOST_GATEWAY" \ + "production sourcing keeps PHLEX_HOST_GATEWAY"; then + tests_failed=1 + fi + fi + + if ! kilo_env_assert_hash_unchanged "$fixture_map" "$fixture_map_before" \ + "carrier relay-map.json hash unchanged after self-test"; then + tests_failed=1 + fi + if ! kilo_env_assert_hash_unchanged "$fixture_env" "$fixture_env_before" \ + "carrier relay-map.env hash unchanged after self-test"; then + tests_failed=1 + fi + if ! kilo_env_assert_hash_unchanged "$fixture_artifact" "$fixture_artifact_before" \ + "carrier artifact-sha256.txt hash unchanged after self-test"; then + tests_failed=1 + fi + + if [ "$tests_failed" -ne 0 ]; then + printf '=== Self-test failed ===\n' >&2 + return 1 + fi + + printf '=== Self-test passed ===\n' + return 0 +} + +# kilo_env_run_self_tests_loaded - runner invoked after helper loaded +# Uses PHLEX_RELAY_FIXTURE_DIR (from environment) instead of argument +kilo_env_run_self_tests_loaded() { + local fixture_root="${PHLEX_RELAY_FIXTURE_DIR-}" + local scratch_root="" + local status=0 + + if [ -z "$fixture_root" ]; then + kilo_env_print_error "PHLEX_RELAY_FIXTURE_DIR must be set when --self-test is used" + return 1 + fi + + if ! scratch_root=$(kilo_env_create_self_test_root); then + kilo_env_print_error "unable to create self-test scratch root" + return 1 + fi + + if kilo_env_run_self_tests_impl "$fixture_root" "$scratch_root"; then + status=0 + else + status=$? + fi + + kilo_env_cleanup_self_test_root "$scratch_root" || return 1 + return "$status" +} diff --git a/.devcontainer/kilo-env.sh b/.devcontainer/kilo-env.sh index 6184a4c16..9440dcd7a 100644 --- a/.devcontainer/kilo-env.sh +++ b/.devcontainer/kilo-env.sh @@ -1,22 +1,576 @@ #!/usr/bin/env bash -# Prepare KILO_CONFIG_CONTENT with rewritten headroom base URLs for container runtime. +# Prepare KILO_CONFIG_CONTENT with rewritten loopback URLs based on relay map. # This script is sourced by /etc/profile.d/ so the variable is available to all processes. +# Conforms to kilo-plan/v5: write-free Kilo configuration rewriting. -# Unset any previous KILO_CONFIG_CONTENT to ensure a clean slate. -unset KILO_CONFIG_CONTENT - -# If the headroom proxy ports (9797, 9798) are active via socat relays, -# Kilo needs a modified config with baseURL pointing to host.docker.internal. -# Parse the host config, rewrite baseURLs, and export as KILO_CONFIG_CONTENT. -if command -v kilo >/dev/null 2>&1 && command -v jq >/dev/null 2>&1; then - # Only rewrite if we have a working Kilo config to work with. - KILO_CONFIG_CONTENT="$(perl -pe 's{("(?:[^"\\]|\\.)*"|'"'"'(?:[^'"'"'\\]|\\.)*'"'"')|//.*}{$1 // ""}ge' "${HOME}/.config/kilo/kilo.jsonc" | jq -c '.provider |= with_entries( - .value.options.baseURL = - (if .value.options.baseURL == null then null - elif (.value.options.baseURL | test("^https?://127\\.0\\.0\\.1:")) - then (.value.options.baseURL | capture("https?://127\\.0\\.0\\.1:(?[0-9]+)(?.*)") | "http://host.docker.internal:" + ((.port | tonumber + 10000) | tostring) + .rest) - else .value.options.baseURL - end) - )')" - export KILO_CONFIG_CONTENT +KILO_ENV_PRODUCTION_ROOT="/run/phlex-host-relays" +KILO_ENV_FIXED_ROOT="$KILO_ENV_PRODUCTION_ROOT" +KILO_ENV_HOST_GATEWAY="host.docker.internal" +KILO_ENV_SELF_TEST_MODE=0 + +# Keep helper lookup independent of later changes to the caller's working directory. +KILO_ENV_SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" || { + printf 'kilo-env: unable to resolve production script path\n' >&2 + [ "${BASH_SOURCE[0]-}" = "$0" ] && exit 1 || return 1 +} +KILO_ENV_SCRIPT_PATH="${KILO_ENV_SCRIPT_PATH}/${BASH_SOURCE[0]##*/}" + +kilo_env_is_sourced() { + [ "${BASH_SOURCE[0]-}" != "$0" ] +} + +kilo_env_print_error() { + printf 'kilo-env: %s\n' "$*" >&2 +} + +kilo_env_require_args() { + local expected="${1-}" + local function_name="${2-}" + [ "$#" -ge 2 ] || return 2 + shift 2 + if [ "$#" -ne "$expected" ]; then + kilo_env_print_error "${function_name} requires ${expected} arguments; got $#" + return 2 + fi +} + +kilo_env_unset_var_quietly() { + kilo_env_require_args 1 "${FUNCNAME[0]}" "$@" || return 2 + local variable_name + variable_name="$1" + unset "$variable_name" 2>/dev/null || true +} + +kilo_env_clear_dynamic_exports() { + local relay_variables + local variable_name + + kilo_env_unset_var_quietly "KILO_CONFIG_CONTENT" + + relay_variables=$(compgen -A variable PHLEX_HOST_RELAY_ 2>/dev/null || true) + if [ -z "$relay_variables" ]; then + return 0 + fi + + while IFS= read -r variable_name; do + if [[ "$variable_name" =~ ^PHLEX_HOST_RELAY_[0-9]+$ ]]; then + kilo_env_unset_var_quietly "$variable_name" + fi + done < None: + raise ValueError(message) + + +def read_text(path_str: str, label: str) -> str: + path = Path(path_str) + try: + return path.read_text(encoding="utf-8") + except FileNotFoundError as exc: + raise ValueError(f"{label} missing: {path}") from exc + except UnicodeDecodeError as exc: + raise ValueError(f"{label} is not valid UTF-8: {path}") from exc + except OSError as exc: + raise ValueError(f"unable to read {label}: {path}: {exc}") from exc + + +def reject_duplicate_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + fail(f"duplicate JSON object key: {key}") + result[key] = value + return result + + +def parse_relay_map(text: str) -> dict[str, str]: + try: + parsed = json.loads(text, object_pairs_hook=reject_duplicate_object) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid relay map JSON: {exc.msg}") from exc + + if not isinstance(parsed, dict): + fail("relay map must be a JSON object") + + canonical = json.dumps(parsed, separators=(",", ":"), sort_keys=True) + if text not in {canonical, canonical + "\n"}: + fail("relay map must be a sorted canonical JSON object") + + relay_map: dict[str, str] = {} + for source_port, relay_port in parsed.items(): + if not isinstance(source_port, str) or not isinstance(relay_port, str): + fail("relay map entries must be string-to-string port mappings") + if source_port not in ALLOWED_RELAYS or ALLOWED_RELAYS[source_port] != relay_port: + fail(f"relay map entry {source_port}={relay_port} is not allowlisted") + relay_map[source_port] = relay_port + + return relay_map + + +def parse_relay_env(text: str, relay_map: dict[str, str]) -> dict[str, str]: + if not relay_map and text in {"", "\n"}: + return {} + + parsed_env: dict[str, str] = {} + for line in text.splitlines(): + match = ENV_PATTERN.fullmatch(line) + if match is None: + fail("relay env must contain only exact PHLEX_HOST_RELAY_= lines") + source_port, relay_port = match.groups() + variable_name = f"PHLEX_HOST_RELAY_{source_port}" + if variable_name in parsed_env: + fail(f"duplicate relay env entry for source port {source_port}") + parsed_env[variable_name] = relay_port + + expected_lines = [ + f"PHLEX_HOST_RELAY_{source_port}={relay_map[source_port]}" + for source_port in sorted(relay_map) + ] + canonical_env = "\n".join(expected_lines) + accepted = {canonical_env} + if canonical_env: + accepted.add(canonical_env + "\n") + if text not in accepted: + fail("relay env must be sorted canonical PHLEX_HOST_RELAY_= lines") + + expected_names = {f"PHLEX_HOST_RELAY_{source_port}" for source_port in relay_map} + if set(parsed_env) != expected_names: + fail("relay env keys do not match relay map keys") + + for source_port, relay_port in relay_map.items(): + variable_name = f"PHLEX_HOST_RELAY_{source_port}" + if parsed_env[variable_name] != relay_port: + fail(f"relay env entry {variable_name} does not match relay map value") + + return parsed_env + + +def strip_jsonc(text: str) -> str: + without_comments: list[str] = [] + in_string = False + escape = False + index = 0 + + while index < len(text): + char = text[index] + + if in_string: + without_comments.append(char) + if escape: + escape = False + elif char == "\\": + escape = True + elif char == '"': + in_string = False + index += 1 + continue + + if char == '"': + in_string = True + without_comments.append(char) + index += 1 + continue + + if char == "/" and index + 1 < len(text): + next_char = text[index + 1] + if next_char == "/": + index += 2 + while index < len(text) and text[index] != "\n": + index += 1 + continue + if next_char == "*": + index += 2 + while index + 1 < len(text) and not (text[index] == "*" and text[index + 1] == "/"): + index += 1 + if index + 1 >= len(text): + fail("unterminated block comment in Kilo config") + index += 2 + continue + + without_comments.append(char) + index += 1 + + if in_string or escape: + fail("unterminated string in Kilo config") + + stripped = "".join(without_comments) + without_trailing_commas: list[str] = [] + in_string = False + escape = False + index = 0 + + while index < len(stripped): + char = stripped[index] + + if in_string: + without_trailing_commas.append(char) + if escape: + escape = False + elif char == "\\": + escape = True + elif char == '"': + in_string = False + index += 1 + continue + + if char == '"': + in_string = True + without_trailing_commas.append(char) + index += 1 + continue + + if char == ",": + look_ahead = index + 1 + while look_ahead < len(stripped) and stripped[look_ahead] in " \t\r\n": + look_ahead += 1 + if look_ahead < len(stripped) and stripped[look_ahead] in "]}": + index += 1 + continue + + without_trailing_commas.append(char) + index += 1 + + if in_string or escape: + fail("unterminated string in Kilo config") + + return "".join(without_trailing_commas) + + +def parse_config(config_path: str) -> object | None: + if not config_path: + return None + + cleaned = strip_jsonc(read_text(config_path, "Kilo config")) + try: + return json.loads(cleaned, object_pairs_hook=reject_duplicate_object) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid Kilo config JSON/JSONC: {exc.msg}") from exc + + +def rewrite_url(value: str, relay_map: dict[str, str], host_gateway: str) -> str: + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https"}: + return value + + hostname = parsed.hostname + if hostname is None or hostname.lower() not in LOOPBACK_HOSTS: + return value + + try: + source_port = parsed.port + except ValueError as exc: + raise ValueError(f"invalid loopback URL port in Kilo config: {value}") from exc + + if source_port is None: + return value + + relay_port = relay_map.get(str(source_port)) + if relay_port is None: + return value + + userinfo = "" + if parsed.username is not None: + userinfo = parsed.username + if parsed.password is not None: + userinfo = f"{userinfo}:{parsed.password}" + userinfo = f"{userinfo}@" + + netloc = f"{userinfo}{host_gateway}:{relay_port}" + return urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment)) + + +def rewrite_value(value: object, relay_map: dict[str, str], host_gateway: str) -> object: + if isinstance(value, dict): + return {key: rewrite_value(item, relay_map, host_gateway) for key, item in value.items()} + if isinstance(value, list): + return [rewrite_value(item, relay_map, host_gateway) for item in value] + if isinstance(value, str): + return rewrite_url(value, relay_map, host_gateway) + return value + + +def main() -> None: + map_path, env_path, config_path, host_gateway = sys.argv[1:5] + relay_map = parse_relay_map(read_text(map_path, "relay map")) + relay_env = parse_relay_env(read_text(env_path, "relay env"), relay_map) + config = parse_config(config_path) + + for source_port in sorted(relay_map): + variable_name = f"PHLEX_HOST_RELAY_{source_port}" + print(f"export\t{variable_name}\t{relay_env[variable_name]}") + + if config is not None: + rewritten = rewrite_value(config, relay_map, host_gateway) + config_content = json.dumps(rewritten, separators=(",", ":"), ensure_ascii=False) + print(f"config\tKILO_CONFIG_CONTENT\t{config_content}") + + +if __name__ == "__main__": + try: + main() + except ValueError as exc: + print(f"kilo-env: {exc}", file=sys.stderr) + raise SystemExit(1) +PY +} + +kilo_env_apply_paths() { + kilo_env_require_args 3 "${FUNCNAME[0]}" "$@" || return 2 + local map_path + local env_path + local config_path + local output + local entry_type + local entry_name + local entry_value + + map_path="$1" + env_path="$2" + config_path="$3" + output="" + + kilo_env_clear_dynamic_exports + + if ! command -v python3 >/dev/null 2>&1; then + kilo_env_print_error "python3 is required to validate relay data" + kilo_env_clear_failed_exports + return 1 + fi + + if ! output=$(kilo_env_collect_exports "$map_path" "$env_path" "$config_path"); then + kilo_env_clear_failed_exports + return 1 + fi + + while IFS=$'\t' read -r entry_type entry_name entry_value; do + if [ -z "$entry_type" ]; then + continue + fi + + case "$entry_type" in + export) + if [[ ! "$entry_name" =~ ^PHLEX_HOST_RELAY_[0-9]+$ ]]; then + kilo_env_print_error "internal export validation failed for ${entry_name}" + kilo_env_clear_failed_exports + return 1 + fi + if ! export "${entry_name}=${entry_value}"; then + kilo_env_print_error "failed to export ${entry_name}" + kilo_env_clear_failed_exports + return 1 + fi + ;; + config) + if [ "$entry_name" != "KILO_CONFIG_CONTENT" ]; then + kilo_env_print_error "internal config validation failed for ${entry_name}" + kilo_env_clear_failed_exports + return 1 + fi + if ! export "KILO_CONFIG_CONTENT=${entry_value}"; then + kilo_env_print_error "failed to export KILO_CONFIG_CONTENT" + kilo_env_clear_failed_exports + return 1 + fi + ;; + *) + kilo_env_print_error "internal helper emitted unsupported record type ${entry_type}" + kilo_env_clear_failed_exports + return 1 + ;; + esac + done <&2 + return 1 +fi + # Run inside the dev container after creation. set -euo pipefail @@ -11,14 +17,18 @@ cat > ~/.actrc <<'EOF' --container-options --userns=keep-id EOF +# Seed host auth into the container-private data volume without modifying the +# read-only host file or replacing credentials already saved in this container. +mkdir -p /root/.local/share/kilo +if [ -f /run/phlex-host-auth.json ] && [ ! -e /root/.local/share/kilo/auth.json ]; then + install -m 0600 /run/phlex-host-auth.json /root/.local/share/kilo/auth.json +fi + # Seed the Kilo Code auth token into the container-private data volume. # The volume is not shared with the host to avoid SQLite conflicts between # the Remote-SSH and devcontainer Kilo Code instances. The API key is # passed in via the KILO_API_KEY remoteEnv variable. if [ -n "${KILO_API_KEY:-}" ]; then - mkdir -p /root/.local/share/kilo - touch /root/.local/share/kilo/auth.json - chmod 0600 /root/.local/share/kilo/auth.json python3 - <<'PY' import json import os @@ -29,15 +39,17 @@ from pathlib import Path # provider keys so Kilo resolves whichever provider a model is routed through. key = os.environ["KILO_API_KEY"] p = Path("/root/.local/share/kilo/auth.json") +auth = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {} +auth.update( + { + "fnal-azure": {"type": "api", "key": key}, + "fnal-ow": {"type": "api", "key": key}, + } +) +p.touch(mode=0o600, exist_ok=True) +p.chmod(0o600) p.write_text( - json.dumps( - { - "fnal-azure": {"type": "api", "key": key}, - "fnal-ow": {"type": "api", "key": key}, - }, - indent=2, - ) - + "\n", + json.dumps(auth, indent=2) + "\n", encoding="utf-8", ) PY diff --git a/.devcontainer/setup-repos.sh b/.devcontainer/setup-repos.sh index 164327fcc..4c0423a5f 100755 --- a/.devcontainer/setup-repos.sh +++ b/.devcontainer/setup-repos.sh @@ -1,4 +1,10 @@ #!/bin/bash + +if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then + printf '%s\n' 'ERROR: setup-repos.sh must be executed, not sourced' >&2 + return 1 +fi + # Clone companion repositories into the codespace so they are available # for reference by developers and AI agents. # diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..bbdabdd7c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +** +!.devcontainer/ +!.devcontainer/Dockerfile +!.devcontainer/gitignore-global +!.devcontainer/kilo-env.sh +!.devcontainer/kilo-env-self-test.sh +!ci/ +!ci/entrypoint.sh diff --git a/.github/workflows/cmake-build.yaml b/.github/workflows/cmake-build.yaml index 9eb7e2f0d..badfcda9e 100644 --- a/.github/workflows/cmake-build.yaml +++ b/.github/workflows/cmake-build.yaml @@ -129,7 +129,7 @@ jobs: ) runs-on: ubuntu-latest outputs: - matrix: ${{ steps.generate.outputs.matrix }} + matrix: ${{ steps.expand-arch.outputs.matrix }} steps: - id: generate uses: Framework-R-D/action-generate-build-matrix@a867325d7153f7fcc7d64fc69eafcb2b97edae39 # v1 @@ -137,6 +137,31 @@ jobs: user-input: ${{ inputs.build-combinations || github.event.inputs.build-combinations }} comment-body: ${{ github.event.comment.body }} + - name: Expand architecture matrix + # Cross each compiler/sanitizer combo with the supported architectures and + # attach the matching runner label. Perfetto profiling runs on both, as the + # CI image installs the native tracebox prebuild for its own architecture. + id: expand-arch + shell: bash + env: + MATRIX_JSON: ${{ steps.generate.outputs.matrix }} + run: | + python3 - <<'PY' + import json + import os + + combos = json.loads(os.environ.get("MATRIX_JSON") or '{"include": []}').get("include", []) + runners = {"amd64": "ubuntu-24.04", "arm64": "ubuntu-24.04-arm"} + + expanded = [] + for combo in combos: + for arch, runner in runners.items(): + expanded.append({**combo, "arch": arch, "runner": runner}) + + with open(os.environ["GITHUB_OUTPUT"], "a") as handle: + print("matrix=" + json.dumps({"include": expanded}), file=handle) + PY + build: needs: [setup, generate-matrix] if: > @@ -146,7 +171,7 @@ jobs: inputs.skip-relevance-check || needs.setup.outputs.has_changes == 'true' ) - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} @@ -277,7 +302,7 @@ jobs: if: matrix.sanitizer == 'perfetto' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: perfetto-traces-${{ matrix.compiler }} + name: perfetto-traces-${{ matrix.compiler }}-${{ matrix.arch }} path: | ${{ needs.setup.outputs.build_path }}/**/*.pftrace retention-days: 30 diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 2dd7b9445..6ab09f3bd 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -73,20 +73,24 @@ jobs: needs: setup if: > always() && (github.event_name == 'workflow_dispatch' || needs.setup.outputs.has_changes == 'true') - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + arch: + - amd64 + - arm64 + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm container: image: ghcr.io/framework-r-d/phlex-ci:latest env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - outputs: - has_coverage_xml: ${{ steps.coverage_outputs.outputs.has_coverage_xml }} - has_coverage_html: ${{ steps.coverage_outputs.outputs.has_coverage_html }} - has_coverage_llvm_info: ${{ steps.coverage_outputs.outputs.has_coverage_llvm_info }} - has_coverage_python_xml: ${{ steps.coverage_outputs.outputs.has_coverage_python_xml }} - artifact_upload_available: ${{ steps.artifact_runtime.outputs.available }} - steps: - name: Determine coverage options id: coverage_options @@ -251,37 +255,6 @@ jobs: echo "::notice::pytest-cov not available; skipping Python coverage report." fi - - name: Generate scripts coverage report - id: report_scripts - if: ${{ steps.report_gcc.outcome != 'skipped' || steps.report_clang.outcome != 'skipped' }} - shell: bash - env: - SOURCE_PATH: ${{ needs.setup.outputs.checkout_path }} - BUILD_PATH: ${{ needs.setup.outputs.build_path }} - run: | - . /entrypoint.sh - SOURCE_DIR="$GITHUB_WORKSPACE/$SOURCE_PATH" - BUILD_DIR="$GITHUB_WORKSPACE/$BUILD_PATH" - - echo "➡️ Generating scripts/ Python coverage report..." - - if ! python3 -c "import pytest_cov" 2>/dev/null; then - echo "::notice::pytest-cov not available; skipping scripts coverage report." - exit 0 - fi - - cd "$SOURCE_DIR" - if python3 -m pytest \ - scripts/test/ \ - --cov=scripts \ - --cov-report=xml:"$BUILD_DIR/coverage-scripts.xml" \ - --cov-report=term-missing \ - -q; then - echo "✅ Scripts coverage report generation succeeded." - else - echo "::warning::Scripts coverage report generation failed (non-fatal)." - fi - - name: Capture coverage artifact availability id: coverage_outputs if: ${{ steps.report_gcc.outcome != 'skipped' || steps.report_clang.outcome != 'skipped' }} @@ -331,8 +304,7 @@ jobs: coverage.xml \ coverage.info \ coverage.info.final \ - coverage-python.xml \ - coverage-scripts.xml; do + coverage-python.xml; do if [ -f "$file" ]; then cp "$file" "$ARTIFACT_DIR/" fi @@ -372,15 +344,17 @@ jobs: steps.artifact_runtime.outputs.available == 'true' && github.actor != 'nektos/act' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: coverage-generated + name: coverage-generated-${{ matrix.arch }} path: coverage-artifacts/ retention-days: 30 coverage-upload: needs: [setup, coverage] - if: - ${{ needs.coverage.result == 'success' && needs.coverage.outputs.artifact_upload_available == 'true' && - github.actor != 'nektos/act' }} + # Run after either matrix success or failure so each successful leg can + # publish its own report without masking a failed architecture. + if: >- + always() && needs.setup.result == 'success' && needs.coverage.result != 'skipped' && needs.coverage.result != + 'cancelled' && github.actor != 'nektos/act' runs-on: ubuntu-latest env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} @@ -397,71 +371,109 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Download coverage bundle + - name: Download amd64 coverage bundle + if: always() + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: coverage-generated-amd64 + path: coverage-artifacts/coverage-generated-amd64 + + - name: Download arm64 coverage bundle + if: always() + continue-on-error: true uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: coverage-generated - path: coverage-artifacts + name: coverage-generated-arm64 + path: coverage-artifacts/coverage-generated-arm64 - - name: Determine coverage files for Codecov + - name: Determine coverage files for each architecture id: codecov_targets + env: + COVERAGE_RESULT: ${{ needs.coverage.result }} run: | set -euo pipefail - files=() - if [ -f coverage-artifacts/coverage.xml ]; then - files+=("coverage-artifacts/coverage.xml") - fi - if [ -f coverage-artifacts/coverage-llvm.info ]; then - files+=("coverage-artifacts/coverage-llvm.info") - fi - if [ -f coverage-artifacts/coverage-python.xml ]; then - files+=("coverage-artifacts/coverage-python.xml") + ls -al coverage-artifacts || true + any_coverage=false + missing_coverage=false + + for arch in amd64 arm64; do + report_dir="coverage-artifacts/coverage-generated-${arch}" + files=() + for report in coverage.xml coverage-llvm.info coverage-python.xml; do + if [ -f "${report_dir}/${report}" ]; then + files+=("${report_dir}/${report}") + fi + done + + if [ "${#files[@]}" -eq 0 ]; then + echo "${arch}_found=false" >> "$GITHUB_OUTPUT" + echo "${arch}_files=" >> "$GITHUB_OUTPUT" + if [ "$COVERAGE_RESULT" = success ]; then + echo "::error::Successful ${arch} coverage job has no downloaded coverage report." + missing_coverage=true + fi + else + any_coverage=true + file_csv=$(IFS=,; printf '%s' "${files[*]}") + echo "Codecov ${arch} upload targets: ${files[*]}" + echo "${arch}_found=true" >> "$GITHUB_OUTPUT" + echo "${arch}_files=${file_csv}" >> "$GITHUB_OUTPUT" + fi + done + + if [ "$any_coverage" = false ]; then + echo "::error::No successful architecture produced a coverage report." + exit 1 fi - ls -al coverage-artifacts || true + has_html=false + html_path="" + for arch in amd64 arm64; do + html_dir="coverage-artifacts/coverage-generated-${arch}/coverage-html" + if [ -d "$html_dir" ]; then + has_html=true + html_path="$html_dir" + break + fi + done + echo "has_html=${has_html}" >> "$GITHUB_OUTPUT" + echo "html_path=${html_path}" >> "$GITHUB_OUTPUT" - if [ "${#files[@]}" -eq 0 ]; then - echo "No coverage files detected; skipping Codecov upload." >&2 - echo "found=false" >> "$GITHUB_OUTPUT" - echo "files=" >> "$GITHUB_OUTPUT" - else - file_csv=$(IFS=,; printf '%s' "${files[*]}") - echo "Codecov upload targets: ${files[*]}" - echo "found=true" >> "$GITHUB_OUTPUT" - echo "files=${file_csv}" >> "$GITHUB_OUTPUT" + if [ "$missing_coverage" = true ]; then + exit 1 fi - - name: Upload coverage to Codecov - if: ${{ steps.codecov_targets.outputs.found == 'true' }} + - name: Upload amd64 coverage to Codecov + if: ${{ always() && steps.codecov_targets.outputs.amd64_found == 'true' }} uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: - files: ${{ steps.codecov_targets.outputs.files }} - flags: unittests - name: phlex-coverage + files: ${{ steps.codecov_targets.outputs.amd64_files }} + flags: amd64 + name: phlex-framework-coverage-amd64 fail_ci_if_error: true verbose: true root_dir: . token: ${{ env.CODECOV_TOKEN }} - - name: Upload scripts coverage to Codecov - if: ${{ hashFiles('coverage-artifacts/coverage-scripts.xml') != '' }} + - name: Upload arm64 coverage to Codecov + if: ${{ always() && steps.codecov_targets.outputs.arm64_found == 'true' }} uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: - files: coverage-artifacts/coverage-scripts.xml - flags: scripts - name: phlex-scripts-coverage - # Non-fatal: scripts coverage is informational (no CI threshold yet). - fail_ci_if_error: false + files: ${{ steps.codecov_targets.outputs.arm64_files }} + flags: arm64 + name: phlex-framework-coverage-arm64 + fail_ci_if_error: true verbose: true root_dir: . token: ${{ env.CODECOV_TOKEN }} - name: Publish HTML coverage report - if: ${{ needs.coverage.outputs.has_coverage_html == 'true' }} + if: ${{ always() && steps.codecov_targets.outputs.has_html == 'true' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-html-report - path: coverage-artifacts/coverage-html/ + path: ${{ steps.codecov_targets.outputs.html_path }}/ if-no-files-found: warn retention-days: 30 @@ -507,9 +519,11 @@ jobs: message: | Coverage workflow completed. - **Result:** ${{ needs.coverage.result == 'success' && '✅ Coverage analysis passed.' - || needs.coverage.result == 'failure' && '❌ Coverage analysis failed.' - || needs.coverage.result == 'cancelled' && '⚠️ Coverage analysis was cancelled.' + **Result:** ${{ (needs.coverage.result == 'failure' || needs.coverage-upload.result == 'failure') + && '❌ Coverage analysis failed.' + || (needs.coverage.result == 'cancelled' || needs.coverage-upload.result == 'cancelled') + && '⚠️ Coverage analysis was cancelled.' + || needs.coverage.result == 'success' && '✅ Coverage analysis passed.' || needs.coverage.result == 'skipped' && 'ℹ️ No relevant changes detected; coverage skipped.' || format('ℹ️ Coverage completed with status: {0}.', needs.coverage.result) }} @@ -521,4 +535,4 @@ jobs: if: always() uses: Framework-R-D/action-complete-pr-comment@5d16749c32fbb7b15503f264dde6e3c98b90b4fb # v2 with: - status: ${{ needs.coverage.result }} + status: ${{ needs.coverage-upload.result == 'failure' && 'failure' || needs.coverage.result }} diff --git a/.github/workflows/python-check.yaml b/.github/workflows/python-check.yaml index 2ad08141f..33376b9fc 100644 --- a/.github/workflows/python-check.yaml +++ b/.github/workflows/python-check.yaml @@ -62,6 +62,10 @@ jobs: file-type: | python yaml + include-globs: | + scripts/** + ci/** + .devcontainer/** head-ref: ${{ inputs.pr-head-sha }} ref: ${{ inputs.ref }} repo: ${{ inputs.repo }} @@ -139,7 +143,18 @@ jobs: inputs.skip-relevance-check || needs.setup.outputs.has_changes == 'true' ) - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + arch: + - amd64 + - arm64 + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} permissions: contents: read env: @@ -164,9 +179,11 @@ jobs: working-directory: ${{ needs.setup.outputs.checkout_path }} run: | echo "➡️ Running scripts/ unit tests with coverage..." - uv run --with pytest --with PyYAML --with pytest-cov \ + uv run --python 3.12 --with pytest --with PyYAML --with pytest-cov \ pytest scripts/test/ -v \ --cov=scripts \ + --cov=ci \ + --cov=.devcontainer \ --cov-report="xml:${GITHUB_WORKSPACE}/coverage-scripts.xml" \ --cov-report=term-missing @@ -175,8 +192,8 @@ jobs: uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: files: coverage-scripts.xml - flags: scripts - name: phlex-scripts-coverage + flags: ${{ matrix.arch }} + name: phlex-scripts-coverage-${{ matrix.arch }} fail_ci_if_error: false verbose: true root_dir: ${{ needs.setup.outputs.checkout_path || '.' }} diff --git a/.gitignore b/.gitignore index deeab8d44..289bf3ba6 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ _codeql_detected_source_root # IDE and editor files .vscode/ +/.devcontainer/docker-compose.host.yml .idea/ *.swp *.swo @@ -59,3 +60,4 @@ docs/build/ .secrets actionlint .coverage +/uv.lock diff --git a/.kilo/kilo.jsonc b/.kilo/kilo.jsonc deleted file mode 100644 index 571a15bec..000000000 --- a/.kilo/kilo.jsonc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "$schema": "https://app.kilo.ai/config.json" -} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eaa736325..aaa525bf2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,14 +43,14 @@ repos: language: system files: \.(h|hpp)$ - id: scripts-unit-tests - name: scripts/ unit tests + name: scripts, CI, and devcontainer unit tests entry: pytest scripts/test/ -v language: python + language_version: python3.12 additional_dependencies: - pytest - PyYAML - files: ^scripts/ - types_or: [python] + files: ^(scripts/|ci/|\.devcontainer/) pass_filenames: false - repo: https://github.com/BlankSpruce/gersemi-pre-commit rev: 0.28.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a98bf484..0a3d6e1b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,6 @@ -cmake_minimum_required(VERSION 3.31) +# Set policies through CMake 4.3 to NEW, avoiding "policy not set" +# warnings while leaving policies from later versions for review. +cmake_minimum_required(VERSION 3.31...4.3) # ############################################################################## # Ensure we have access to a suitable version of Cetmodules diff --git a/CMakePresets.json b/CMakePresets.json index 6ccd5cd7f..2ef3cbd00 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -9,6 +9,7 @@ "CMAKE_CXX_STANDARD": "23", "CMAKE_CXX_STANDARD_REQUIRED": "YES", "CMAKE_CXX_EXTENSIONS": "NO", + "CMAKE_POLICY_DEFAULT_CMP0218": "OLD", "CMAKE_PROJECT_TOP_LEVEL_INCLUDES": "CetProvideDependency" } }, diff --git a/ci/Dockerfile b/ci/Dockerfile index 5655b5416..4a280ada4 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -1,23 +1,74 @@ # syntax=docker/dockerfile:1.4 -# Multi-target Dockerfile for building the Phlex CI and development images. +# Multi-target podman/Dockerfile for building the Phlex CI and +# development images. +######################################################################## # Podman instructions for building tagged images with this file: # -# $ podman build --format docker \ -# [-v :/build-cache:Z] \ -# [-v ~/.gnupg:/run/gpg:Z] \ -# --target ci --tag phlex-ci: . -# $ podman build --format docker \ -# [-v :/build-cache:Z] \ -# [-v ~/.gnupg:/run/gpg:Z] \ -# --target dev --tag phlex-dev: . -# $ podman login ghcr.io --username -p -# $ podman push phlex-ci: ghcr.io/framework-r-d/phlex-ci: -# $ podman push phlex-dev: ghcr.io/framework-r-d/phlex-dev: -# ... and optionally push either image as "latest". +# Customizable variables for substitution in commands below: +# +# GPG_KEY_ID= +# PHLEX_TAG_EXTRA= +# +# PHLEX_DEFAULT_COMPILER=gcc +# PHLEX_GCC_VERSION=15.3 +# PHLEX_LLVM_TARGET=auto +# PHLEX_LLVM_VERSION=22.1.8 +# PHLEX_PODMAN_TARGET=ci|dev +# PHLEX_SPACK_CACHE= # /scratch/${USER}/phlex-build-cache... +# PHLEX_SPACK_TARGET=x86_64_v3|aarch64 +# PHLEX_TAG_BASE=phlex-${PHLEX_PODMAN_TARGET} +# PHLEX_TAG_TAG=$(date +%F)${PHLEX_TAG_EXTRA} +# PHLEX_TAG_REMOTE=ghcr.io/framework-r-d/ +# +# Derivative variables: +# +# PHLEX_TAG_LOCAL=${PHLEX_TAG_BASE}:${PHLEX_TAG_TAG}-${PHLEX_SPACK_TARGET} +# PHLEX_TAG_LATEST=${PHLEX_TAG_REMOTE}${PHLEX_TAG_BASE}:latest +# PHLEX_TAG_MANIFEST=${PHLEX_TAG_REMOTE}${PHLEX_TAG_BASE}:${PHLEX_TAG_TAG} +# +# Docker build command: +# +# podman build --format docker \ +# ${PHLEX_SPACK_CACHE:+-v ${PHLEX_SPACK_CACHE}:/build-cache:Z} \ +# -v ~/.gnupg:/run/gpg:Z \ +# --target ${PHLEX_PODMAN_TARGET} --tag ${PHLEX_TAG_LOCAL} \ +# ${GPG_KEY_ID:+--build-arg GPG_KEY_ID=${GPG_KEY_ID}} \ +# --build-arg PHLEX_SPACK_TARGET=${PHLEX_SPACK_TARGET} \ +# --build-arg PHLEX_LLVM_TARGET=${PHLEX_LLVM_TARGET} \ +# --build-arg PHLEX_GCC_VERSION=${PHLEX_GCC_VERSION} \ +# --build-arg PHLEX_LLVM_VERSION=${PHLEX_LLVM_VERSION} \ +# --build-arg PHLEX_DEFAULT_COMPILER=${PHLEX_DEFAULT_COMPILER} \ +# ci/ +# +# Registry login command: +# +# podman login ghcr.io --username -p +# +# Push image to registry: +# +# podman push ${PHLEX_TAG_LOCAL} ${PHLEX_TAG_REMOTE}${PHLEX_TAG_LOCAL} +# +# After both architectures have been pushed: +# +# podman manifest create ${PHLEX_TAG_MANIFEST} # -# where is the date (e.g. "2025-08-12"). +# For each architecture-specific ${PHLEX_TAG_LOCAL}: +# +# podman manifest add ${PHLEX_TAG_MANIFEST} ${PHLEX_TAG_REMOTE}${PHLEX_TAG_LOCAL} +# +# Inspect and push the full manifest: +# +# podman manifest inspect ${PHLEX_TAG_MANIFEST} +# podman manifest push --all ${PHLEX_TAG_MANIFEST} +# +# If you are ready to turn over the latest tag, then: +# +# podman tag ${PHLEX_TAG_MANIFEST} ${PHLEX_TAG_LATEST} +# podman push ${PHLEX_TAG_LATEST} +# +######################################################################## # # Binary package signing (optional): # When -v :/build-cache is supplied, newly built @@ -48,17 +99,59 @@ # Host prerequisite: systemctl --user enable --now podman.socket ARG GPG_KEY_ID + FROM ubuntu:24.04 AS base +# Build arguments +ARG PHLEX_SPACK_TARGET=auto +ARG PHLEX_LLVM_TARGET=auto +ARG PHLEX_GCC_VERSION=15.3 +ARG PHLEX_LLVM_VERSION=22.1.8 +ARG PHLEX_DEFAULT_COMPILER=gcc + ENV DEBIAN_FRONTEND=noninteractive ENV LANG=en_US.UTF-8 ENV LC_ALL=en_US.UTF-8 ENV SPACK_USER_CONFIG_PATH=/dev/null ENV SPACK_DISABLE_LOCAL_CONFIG=true ENV PHLEX_SPACK_ENV=/opt/spack-environments/phlex-ci +ENV PHLEX_SPACK_SOURCE_TARGET=${PHLEX_SPACK_TARGET} +ENV PHLEX_LLVM_SOURCE_TARGET=${PHLEX_LLVM_TARGET} +ENV PHLEX_GCC_VERSION=${PHLEX_GCC_VERSION} +ENV PHLEX_LLVM_VERSION=${PHLEX_LLVM_VERSION} +ENV PHLEX_DEFAULT_COMPILER=${PHLEX_DEFAULT_COMPILER} SHELL ["/bin/bash", "-c"] +# Validate the raw build inputs before any Spack clone or configuration. +RUN <<'ARCH_MAP_VALIDATE' +set -euo pipefail +case "$PHLEX_SPACK_SOURCE_TARGET" in + auto|amd64|arm64|x86_64_v3|aarch64) ;; + *) + echo "ERROR: PHLEX_SPACK_TARGET must be 'amd64', 'arm64', 'x86_64_v3', or 'aarch64'" >&2 + echo " Got: '$PHLEX_SPACK_SOURCE_TARGET'" >&2 + exit 1 + ;; +esac +case "$PHLEX_LLVM_SOURCE_TARGET" in + auto|x86|aarch64) ;; + *) + echo "ERROR: PHLEX_LLVM_TARGET must be 'auto', 'x86', or 'aarch64'" >&2 + echo " Got: '$PHLEX_LLVM_SOURCE_TARGET'" >&2 + exit 1 + ;; +esac +case "$PHLEX_DEFAULT_COMPILER" in + gcc|clang) ;; + *) + echo "ERROR: PHLEX_DEFAULT_COMPILER must be 'gcc' or 'clang'" >&2 + echo " Got: '$PHLEX_DEFAULT_COMPILER'" >&2 + exit 1 + ;; +esac +ARCH_MAP_VALIDATE + ######################################################################## # Install essential build tools, git, and Python @@ -84,6 +177,8 @@ apt-get install -y --no-install-recommends \ lsb-release \ ninja-build \ pkg-config \ + rpm \ + rpm2cpio \ software-properties-common \ sudo \ unzip \ @@ -94,6 +189,88 @@ apt-get clean rm -rf /var/lib/apt/lists/* INSTALL_ESSENTIALS +######################################################################## +# Validate and preserve architecture and build details +RUN <<'ARCHITECTURE_CHECK' +set -euo pipefail + +# Validate that the container build host architecture matches the requested SPACK target +# This check runs early, before any Spack clone/install/concretization operations +NATIVE_ARCH=$(dpkg --print-architecture) +case "$NATIVE_ARCH" in + amd64) if [ "$PHLEX_SPACK_SOURCE_TARGET" != "auto" ] && [ "$PHLEX_SPACK_SOURCE_TARGET" != "amd64" ] && [ "$PHLEX_SPACK_SOURCE_TARGET" != "x86_64_v3" ]; then + echo "ERROR: Host architecture '$NATIVE_ARCH' does not match PHLEX_SPACK_TARGET='$PHLEX_SPACK_SOURCE_TARGET'" >&2 + echo " CI images must be built for the host architecture" >&2 + exit 1 + fi + # Normalize amd64 to x86_64_v3 for Spack + normalized_spack_target=x86_64_v3 + normalized_llvm_target=x86 + ;; + arm64) if [ "$PHLEX_SPACK_SOURCE_TARGET" != "auto" ] && [ "$PHLEX_SPACK_SOURCE_TARGET" != "arm64" ] && [ "$PHLEX_SPACK_SOURCE_TARGET" != "aarch64" ]; then + echo "ERROR: Host architecture '$NATIVE_ARCH' does not match PHLEX_SPACK_TARGET='$PHLEX_SPACK_SOURCE_TARGET'" >&2 + echo " CI images must be built for the host architecture" >&2 + exit 1 + fi + # Normalize arm64 to aarch64 for Spack + normalized_spack_target=aarch64 + normalized_llvm_target=aarch64 + ;; + *) + echo "ERROR: Unknown host architecture '$NATIVE_ARCH'" >&2 + exit 1 + ;; +esac + +if [ "$PHLEX_LLVM_SOURCE_TARGET" != "auto" ] && + [ "$PHLEX_LLVM_SOURCE_TARGET" != "$normalized_llvm_target" ]; then + echo "ERROR: Host architecture '$NATIVE_ARCH' does not match PHLEX_LLVM_TARGET='$PHLEX_LLVM_SOURCE_TARGET'" >&2 + echo " Expected LLVM target '$normalized_llvm_target' for PHLEX_SPACK_TARGET='$PHLEX_SPACK_SOURCE_TARGET'" >&2 + exit 1 +fi + +# RUN layers do not retain shell exports. Persist normalized values for later +# Spack operations instead of relying on Docker layer state. +cat > /etc/profile.d/phlex-targets.sh < /opt/render-spack-template.sh <<'RENDER_SPACK_TEMPLATE' +render_spack_template() { + if [ "$#" -ne 2 ]; then + echo "ERROR: render_spack_template requires template, output" >&2 + return 2 + fi + + local template_path=$1 + local output_path=$2 + + local variable_name + for variable_name in PHLEX_GCC_VERSION PHLEX_LLVM_TARGET PHLEX_LLVM_VERSION \ + PHLEX_NATIVE_GCC_VERSION PHLEX_SPACK_TARGET; do + if [ -z "${!variable_name:-}" ]; then + echo "ERROR: $variable_name must be set before rendering" >&2 + return 1 + fi + done + + sed -E \ + -e "s|\\$\\{PHLEX_GCC_VERSION\\}|${PHLEX_GCC_VERSION}|g" \ + -e "s|\\$\\{PHLEX_LLVM_TARGET\\}|${PHLEX_LLVM_TARGET}|g" \ + -e "s|\\$\\{PHLEX_LLVM_VERSION\\}|${PHLEX_LLVM_VERSION}|g" \ + -e "s|\\$\\{PHLEX_NATIVE_GCC_VERSION\\}|${PHLEX_NATIVE_GCC_VERSION}|g" \ + -e "s|\\$\\{PHLEX_SPACK_TARGET\\}|${PHLEX_SPACK_TARGET}|g" \ + "$template_path" > "$output_path" +} +RENDER_SPACK_TEMPLATE +chmod 644 /opt/render-spack-template.sh +ARCHITECTURE_CHECK + ######################################################################## # Clone Spack checkout used by both CI and dev images @@ -121,7 +298,7 @@ PREP_SPACK_GROUP ######################################################################## # Configure Spack: add repos, compilers, externals, and cache locations -COPY packages.yaml /tmp/packages.yaml +COPY packages.yaml /tmp/packages.yaml.template COPY upgrade_repos.py /tmp/upgrade_repos.py ENV GNUPGHOME=/run/gpg @@ -141,6 +318,8 @@ chmod -R g+rwX "$SPACK_REPO_ROOT" find "$SPACK_REPO_ROOT" -type d -exec chmod g+s {} + . /spack/share/spack/setup-env.sh +. /etc/profile.d/phlex-targets.sh +. /opt/render-spack-template.sh # Other config settings spack config --scope defaults add config:build_stage:/opt/spack-stage @@ -151,9 +330,10 @@ spack config --scope defaults add config:install_tree:padded_length:255 # Find compilers (destination packages.yaml for Spack >=1.0) spack --timestamp compiler find --scope site -# Install settings from our copied packages.yaml +# Install settings from the rendered build-time packages.yaml copy. +render_spack_template /tmp/packages.yaml.template /tmp/packages.yaml spack config --scope site add -f /tmp/packages.yaml -rm -f /tmp/packages.yaml +rm -f /tmp/packages.yaml /tmp/packages.yaml.template # Discover externals # @@ -174,7 +354,6 @@ rm -f /tmp/upgrade_repos.py spack repo list spack clean -m spack audit configs - SPACK_CONFIGURE_DEFAULTS ######################################################################## @@ -190,6 +369,7 @@ set -euo pipefail # configured mirrors so that signed packages from those mirrors can be # verified during subsequent installs. . /spack/share/spack/setup-env.sh +. /etc/profile.d/phlex-targets.sh spack mirror add --type binary phlex-ci-scisoft https://scisoft.fnal.gov/scisoft/spack-packages/phlex-dev @@ -210,6 +390,7 @@ set -euo pipefail # was cloned above. If spack versions -n lists it as "new" (not checksummed), fetch the archive # and add its checksum to the package file so that Spack can install it. . /spack/share/spack/setup-env.sh +. /etc/profile.d/phlex-targets.sh if spack versions -n intel-tbb | grep -Fwq '2023.0.0'; then EDITOR=true spack checksum -ab intel-tbb 2023.0.0 @@ -224,14 +405,19 @@ set -euo pipefail # Install GCC outside the Phlex development environment. . /spack/share/spack/setup-env.sh +. /etc/profile.d/phlex-targets.sh + +# PHLEX_SPACK_TARGET is already normalized to x86_64_v3 or aarch64 by the +# early architecture check (before this layer runs). +GCC_TARGET="${PHLEX_SPACK_TARGET}" spack --timestamp install --fail-fast -j "$(nproc)" \ --no-check-signature \ - gcc@15.3 target=x86_64_v3 \ + gcc@${PHLEX_GCC_VERSION} target="${GCC_TARGET}" \ +binutils+graphite~nvptx+piclibs+strip \ build_type=Release \ languages=c,c++,fortran,lto \ - %c,cxx=gcc@13.3.0 # % must be LAST (along with ^ dependencies) + %c,cxx=gcc@${PHLEX_NATIVE_GCC_VERSION} # % must be LAST (along with ^ dependencies) if [ -d "/build-cache" ]; then key_id=${GPG_KEY_ID:-$({ gpg --list-secret-keys --with-colons | grep -Ee '^sec' | cut -d: -f 5 | head -n1; } 2>/dev/null || true)} @@ -240,13 +426,13 @@ if [ -d "/build-cache" ]; then else key_args=(-u) fi - spack --timestamp buildcache create --allow-missing ${key_args[*]:+"${key_args[@]}"} -j $(nproc) --update-index \ - /build-cache $(spack find --no-groups -LI | \ - sed -Ene 's&^\[\+\] +([^[:space:]]+).*$&/\1&p') + spack --timestamp buildcache create --allow-missing ${key_args[*]:+"${key_args[@]}"} -j $(nproc) \ + --update-index /build-cache $(spack find --no-groups -LI | \ + sed -Ene 's&^\[\+\] +([^[:space:]]+).*$&/\1&p') fi # Make GCC external to simplify concretization -spack compiler find $(spack location -i gcc@15) +spack compiler find $(spack location -i gcc@${PHLEX_GCC_VERSION}) spack clean -dfs SPACK_INSTALL_GCC @@ -254,22 +440,31 @@ SPACK_INSTALL_GCC ######################################################################## # Create, activate, and concretize the phlex-ci Spack environment -COPY spack.yaml /tmp/spack.yaml +COPY spack.yaml /tmp/spack.yaml.template +COPY promote_phlex_dependencies.py /tmp/promote_phlex_dependencies.py RUN <<'SPACK_CONCRETIZE_ENV' set -euo pipefail # Build the phlex-ci environment lockfile with consistent permissions +# PHLEX_SPACK_TARGET and PHLEX_LLVM_TARGET are already normalized by +# the early architecture check before this layer runs. + . /spack/share/spack/setup-env.sh +. /etc/profile.d/phlex-targets.sh +. /opt/render-spack-template.sh +render_spack_template /tmp/spack.yaml.template /tmp/spack.yaml mkdir -p "$(dirname "$PHLEX_SPACK_ENV")" rm -rf "$PHLEX_SPACK_ENV" spack --timestamp env create -d "$PHLEX_SPACK_ENV" /tmp/spack.yaml -rm -f /tmp/spack.yaml spack --timestamp env activate -d "$PHLEX_SPACK_ENV" spack --timestamp concretize +# Phlex itself is not installed; promote its dependencies so activation reaches their hooks. +spack python /tmp/promote_phlex_dependencies.py chgrp -R spack "$PHLEX_SPACK_ENV" chmod -R g+rwX "$PHLEX_SPACK_ENV" find "$PHLEX_SPACK_ENV" -type d -exec chmod g+s {} + +rm -f /tmp/spack.yaml /tmp/spack.yaml.template /tmp/promote_phlex_dependencies.py SPACK_CONCRETIZE_ENV ######################################################################## @@ -288,7 +483,6 @@ set -euo pipefail rm -rf "$PHLEX_SPACK_ENV/.spack-env/view" rm -rf "$PHLEX_SPACK_ENV"/.spack-env/view.old.* - spack --timestamp install --fail-fast -j $(nproc) \ --no-add --only-concrete --no-check-signature \ cmake lcov ninja py-gcovr py-pytest-cov py-pyyaml @@ -343,8 +537,12 @@ set -euo pipefail rm -rf "$PHLEX_SPACK_ENV/.spack-env/view" rm -rf "$PHLEX_SPACK_ENV"/.spack-env/view.old.* +# PHLEX_LLVM_TARGET is sourced from the normalized carrier prepared early in the build +# Valid values: x86 (for x86_64_v3), aarch64 (for aarch64) + spack --timestamp install --fail-fast -j $(nproc) \ - --no-add --only-concrete --no-check-signature llvm + --no-add --only-concrete --no-check-signature \ + llvm@${PHLEX_LLVM_VERSION} +zstd +llvm_dylib +link_llvm_dylib ~lldb targets="${PHLEX_LLVM_TARGET}" if [ -d "/build-cache" ]; then key_id=${GPG_KEY_ID:-$({ gpg --list-secret-keys --with-colons | grep -Ee '^sec' | cut -d: -f 5 | head -n1; } 2>/dev/null || true)} @@ -455,6 +653,7 @@ WRITE_BUILDCACHE # Install Perfetto SDK for profiling ARG INSTALL_TRACEBOX=true +ARG INSTALL_TRACEBOX_ARM64=true RUN <<'INSTALL_PERFETTO' set -euo pipefail @@ -480,14 +679,38 @@ echo "${PERFETTO_CC_SHA256} perfetto.cc" | sha256sum -c - chmod 644 perfetto.h perfetto.cc if [ "${INSTALL_TRACEBOX}" = "true" ]; then - # tracebox is downloaded from get.perfetto.dev/tracebox, which currently - # serves the latest binary with no versioned or checksum-verifiable download - # offered by the Perfetto project. Opt out with --build-arg INSTALL_TRACEBOX=false - # if supply-chain policy requires a verifiable artifact. Revisit when the - # project provides versioned tracebox releases with integrity metadata. - curl -fsSL -o tracebox https://get.perfetto.dev/tracebox - chmod +x tracebox - mv tracebox /usr/local/bin/ + # Perfetto publishes versioned tracebox prebuilts with SHA-256 checksums, as + # listed in the tracebox prebuilt manifest of tools/record_android_trace + # (generated by tools/release/roll-prebuilts). Pin the release and the + # per-architecture checksums here; update both when rolling the SDK version. + PERFETTO_TRACEBOX_VERSION=v58.2 + TRACEBOX_BASE=https://commondatastorage.googleapis.com/perfetto-luci-artifacts + ARCH=$(dpkg --print-architecture) + TRACEBOX_ARCH_DIR="" + case "$ARCH" in + amd64) + TRACEBOX_ARCH_DIR=linux-amd64 + TRACEBOX_SHA256=5348e5f3513dec2e70f02d3e4296cfed546e6ebc7f9a7cc6b5499f363b4aea03 + ;; + arm64) + if [ "${INSTALL_TRACEBOX_ARM64}" != "true" ]; then + echo "tracebox arm64 installation disabled via --build-arg INSTALL_TRACEBOX_ARM64=false" + else + TRACEBOX_ARCH_DIR=linux-arm64 + TRACEBOX_SHA256=2d21883e9c55818f851fd8c391afb9b366032b3110076b4fd41f31f30b5849fa + fi + ;; + *) + echo "ERROR: Unsupported architecture '$ARCH' for tracebox download" >&2 + exit 1 + ;; + esac + if [ -n "${TRACEBOX_ARCH_DIR}" ]; then + curl -fsSL -o tracebox "${TRACEBOX_BASE}/${PERFETTO_TRACEBOX_VERSION}/${TRACEBOX_ARCH_DIR}/tracebox" + echo "${TRACEBOX_SHA256} tracebox" | sha256sum -c - + chmod +x tracebox + mv tracebox /usr/local/bin/ + fi fi INSTALL_PERFETTO @@ -502,7 +725,9 @@ USER root # Dev image additions (tooling and dev user configuration) FROM base AS dev +ARG PHLEX_DEFAULT_COMPILER=clang ENV SPACK_GROUP=spack +ENV PHLEX_DEFAULT_COMPILER=${PHLEX_DEFAULT_COMPILER} ######################################################################## # Install useful developer command-line tools with apt-get @@ -562,18 +787,47 @@ PREP_DEV_WORKSPACE ######################################################################## # Install GitHub's act CLI for local workflow testing +ARG INSTALL_ACT=true +ARG INSTALL_ACT_ARM64=true + RUN <<'INSTALL_ACT' set -euo pipefail # Install GitHub's act CLI for local workflow testing -download_url=$(curl -s https://api.github.com/repos/nektos/act/releases/latest | \ - grep -Ee '"browser_download_url": .*/act_Linux_x86_64\.' | \ - cut -d '"' -f 4) -if [ -z "$download_url" ]; then - echo "Failed to determine act download URL" >&2 - exit 1 +# Pin the release and its architecture-specific checksums together. +ACT_VERSION=v0.2.89 + +if [ "${INSTALL_ACT}" != "true" ]; then + echo "act installation disabled via --build-arg INSTALL_ACT=false" + exit 0 fi -curl -sfSL "$download_url" | tar -C /usr/local/bin -zx act + +ARCH=$(dpkg --print-architecture) +ARCHIVE="" +case "$ARCH" in + amd64) + ARCHIVE="Linux_x86_64" + ACT_SHA256=0191d6f1f3b716b5c55820032605d05fc3c1cdbf581ebeff655019e5dd1524c0 + ;; + arm64) + if [ "${INSTALL_ACT_ARM64}" != "true" ]; then + echo "act arm64 installation disabled via --build-arg INSTALL_ACT_ARM64=false" + exit 0 + fi + ARCHIVE="Linux_arm64" + ACT_SHA256=daa8679ba9615a74d2d0cec321dc593f21948a2a11bb65862b063d8b930f4bcb + ;; + *) + echo "ERROR: Unsupported architecture '$ARCH' for act download" >&2 + exit 1 + ;; +esac + +act_archive=$(mktemp) +trap 'rm -f "$act_archive"' EXIT +curl -fsSL -o "$act_archive" "https://github.com/nektos/act/releases/download/${ACT_VERSION}/act_${ARCHIVE}.tar.gz" +echo "${ACT_SHA256} ${act_archive}" | sha256sum -c - +tar -C /usr/local/bin -zxf "$act_archive" act chmod +x /usr/local/bin/act INSTALL_ACT diff --git a/ci/entrypoint.sh b/ci/entrypoint.sh index b410fff9f..def0f00cb 100644 --- a/ci/entrypoint.sh +++ b/ci/entrypoint.sh @@ -1,15 +1,83 @@ #!/bin/bash +# vi: set ft=sh b:is_bash=1 fenc=utf-8 : +# -*- Local Variables: +# -*- mode: shell-script +# -*- coding: utf-8 +# -*- sh-shell: bash +# -*- End: + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + printf '%s\n' 'ERROR: entrypoint.sh must be sourced, not executed' >&2 + exit 1 +fi # Disable Spack's user scope to prevent user-level config interference export SPACK_USER_CONFIG_PATH=/dev/null export SPACK_DISABLE_LOCAL_CONFIG=true +if [ -f /etc/profile.d/phlex-targets.sh ]; then + . /etc/profile.d/phlex-targets.sh || return 1 +fi + : "${PHLEX_SPACK_ENV:=/opt/spack-environments/phlex-ci}" +# PHLEX_SPACK_TARGET must be normalized to x86_64_v3 or aarch64 (Dockerfile handles this) +# Default is x86_64_v3 for backward compatibility +: "${PHLEX_SPACK_TARGET:=x86_64_v3}" +# CI images default to GCC; developer images use Clang with the same GCC runtime. +: "${PHLEX_DEFAULT_COMPILER:=gcc}" + +# Validate PHLEX_SPACK_TARGET is normalized (x86_64_v3 or aarch64) +# Dockerfile normalizes amd64->x86_64_v3 and arm64->aarch64 before this runs +case "$PHLEX_SPACK_TARGET" in + x86_64_v3) + : "${PHLEX_LLVM_TARGET:=x86}" + ;; + aarch64) + : "${PHLEX_LLVM_TARGET:=aarch64}" + ;; + *) + echo "ERROR: PHLEX_SPACK_TARGET must be 'x86_64_v3' or 'aarch64'" >&2 + echo " Got: '$PHLEX_SPACK_TARGET'" >&2 + return 1 + ;; +esac + +# Validate that SPACK target and LLVM target are compatible (no x86_64_v3 + AArch64, no aarch64 + x86) +case "$PHLEX_SPACK_TARGET-$PHLEX_LLVM_TARGET" in + *-auto|x86_64_v3-x86|aarch64-aarch64) ;; + *) + echo "ERROR: Host/target mismatch: PHLEX_SPACK_TARGET=$PHLEX_SPACK_TARGET with PHLEX_LLVM_TARGET=$PHLEX_LLVM_TARGET" >&2 + return 1 + ;; +esac + +case "$PHLEX_DEFAULT_COMPILER" in + gcc|clang) ;; + *) + echo "ERROR: PHLEX_DEFAULT_COMPILER must be 'gcc' or 'clang'" >&2 + echo " Got: '$PHLEX_DEFAULT_COMPILER'" >&2 + return 1 + ;; +esac + +. /spack/share/spack/setup-env.sh || return 1 +spack env activate -d "$PHLEX_SPACK_ENV" || return 1 + +# Resolve the bootstrapped compiler outside the active environment, where it +# may appear only as an external. Prefer the image's recorded versions; older +# images without a carrier must have a unique matching installation. +gcc_path=$(spack -E location -i \ + "gcc${PHLEX_GCC_VERSION:+@${PHLEX_GCC_VERSION}}" \ + "%c,cxx=gcc${PHLEX_NATIVE_GCC_VERSION:+@${PHLEX_NATIVE_GCC_VERSION}}") || return 1 -. /spack/share/spack/setup-env.sh -spack env activate -d "$PHLEX_SPACK_ENV" -PATH=$(spack -E location -i gcc@15 %c,cxx=gcc@13)/bin:$PATH -# spack env activate sets CC/CXX to clang/clang++ because LLVM is an explicit -# spec that provides compiler virtuals. Override so that builds which do not -# name a compiler explicitly (e.g. the CMake "default" preset) use GCC 15. -export CC=gcc CXX=g++ +if [ "$PHLEX_DEFAULT_COMPILER" = "gcc" ]; then + PATH="$gcc_path/bin:${PATH-}" + export CC=gcc CXX=g++ +else + # Spack's Clang config sets --gcc-install-dir, which makes --gcc-toolchain + # unused (and fatal under -Werror). Bind the exact installation instead. + gcc_libgcc=$("$gcc_path/bin/gcc" -print-libgcc-file-name) || return 1 + export CC=clang CXX=clang++ + export CFLAGS="--gcc-install-dir=${gcc_libgcc%/*}" + export CXXFLAGS="--gcc-install-dir=${gcc_libgcc%/*}" +fi diff --git a/ci/packages.yaml b/ci/packages.yaml index 7e7db4307..a520e43ab 100644 --- a/ci/packages.yaml +++ b/ci/packages.yaml @@ -6,27 +6,30 @@ # needing to repeat them in every spack.yaml. # # Design notes: -# - All packages default to target x86_64_v3 and compiler gcc@13. -# This covers root's own dependencies (e.g. zlib, python, xrootd) -# which do not need to share the gcc@15 ABI of root itself. -# - root and its dependents (phlex) require gcc@15; those overrides +# - All packages default to target derived from PHLEX_SPACK_TARGET (amd64->x86_64_v3, arm64->aarch64) +# and compiler gcc@${PHLEX_NATIVE_GCC_VERSION}. This covers root's own dependencies (e.g. zlib, python, xrootd) +# which do not need to share the gcc@${PHLEX_GCC_VERSION} ABI of root itself. +# - root and its dependents (phlex) require gcc@${PHLEX_GCC_VERSION}; those overrides # live in each environment's spack.yaml, not here. # - System zlib is declared external so Spack does not build its own # zlib / zlib-ng, eliminating duplicate virtual-provider builds. +# - ${PHLEX_SPACK_TARGET} is substituted at build time with the normalized +# Spack target ("x86_64_v3" or "aarch64"). packages: all: require: - - target=x86_64_v3 + - target=${PHLEX_SPACK_TARGET} providers: + "tbb:": [intel-tbb, intel-oneapi-tbb] "zlib-api:": [zlib] c: - prefer: ["gcc@13"] + prefer: ["gcc@${PHLEX_NATIVE_GCC_VERSION}"] cxx: - prefer: ["gcc@13"] + prefer: ["gcc@${PHLEX_NATIVE_GCC_VERSION}"] fortran: - prefer: ["gcc@13"] + prefer: ["gcc@${PHLEX_NATIVE_GCC_VERSION}"] # Use the system zlib rather than building zlib / zlib-ng from source. # The external entry must match the Ubuntu 24.04 package layout. diff --git a/ci/promote_phlex_dependencies.py b/ci/promote_phlex_dependencies.py new file mode 100644 index 000000000..1886785e5 --- /dev/null +++ b/ci/promote_phlex_dependencies.py @@ -0,0 +1,84 @@ +#!/usr/bin/env spack python +"""Promote locked Phlex runtime dependencies to roots without re-concretizing.""" + +from __future__ import annotations + +import inspect +import os +import sys + + +def promote_dependencies(env) -> list[str]: + """Register existing link/run dependencies as abstract and concrete roots.""" + from spack.spec import Spec + + phlex_roots = [spec for spec in env.concrete_roots() if spec.name == "phlex"] + if len(phlex_roots) != 1: + raise RuntimeError("Expected exactly one concrete Phlex root in the environment") + phlex = phlex_roots[0] + + # This core API is not covered by Spack's versioned Package API contract. + method = getattr(env, "add_concrete_spec", None) + try: + if not callable(method): + raise TypeError("add_concrete_spec is missing or not callable") + signature = inspect.signature(method) + signature.bind(spec=phlex, concrete=phlex, new=True) + except (TypeError, ValueError) as exc: + raise RuntimeError( + "Unsupported Spack Environment.add_concrete_spec() API: expected a callable " + "accepting spec, concrete, and new=True with no other required arguments. " + "Review the promotion script against this Spack revision before rebuilding." + ) from exc + + hashes = {spec.dag_hash() for spec in env.all_specs()} + roots = {spec.dag_hash() for spec in env.concrete_roots()} + dependencies = phlex.dependencies(deptype=("link", "run")) + promoted = [] + + with env.write_transaction(): + for dep in dependencies: + dep_hash = dep.dag_hash() + if dep_hash in roots: + continue + user_spec = Spec(f"{dep.name}/{dep_hash}") + if not env.add(user_spec): + raise RuntimeError(f"Dependency {user_spec} is already an unconcretized root") + env.add_concrete_spec(spec=user_spec, concrete=dep, new=True) + roots.add(dep_hash) + promoted.append(dep.name) + + if {spec.dag_hash() for spec in env.concrete_roots()} != roots: + raise RuntimeError("Spack add_concrete_spec() did not register the expected roots") + if {spec.dag_hash() for spec in env.all_specs()} != hashes: + raise RuntimeError("Dependency promotion unexpectedly changed the concrete DAG") + + # Installation layers regenerate the view after dependencies are installed. + env.write(regenerate=False) + + return promoted + + +def main() -> None: + """Update the image's locked environment and verify the persisted roots.""" + try: + import spack.environment as ev + + env_path = os.environ["PHLEX_SPACK_ENV"] + env = ev.Environment(env_path) + promoted = promote_dependencies(env) + expected_roots = {spec.dag_hash() for spec in env.concrete_roots()} + expected_hashes = {spec.dag_hash() for spec in env.all_specs()} + persisted = ev.Environment(env_path) + if {spec.dag_hash() for spec in persisted.concrete_roots()} != expected_roots: + raise RuntimeError("Spack did not persist the promoted dependency roots") + if {spec.dag_hash() for spec in persisted.all_specs()} != expected_hashes: + raise RuntimeError("Spack changed the concrete DAG while writing the environment") + except Exception as exc: + sys.exit(f"ERROR: Cannot promote Phlex dependency roots: {exc}") + + print("Promoted Phlex dependencies: " + (", ".join(promoted) or "none (already roots)")) + + +if __name__ == "__main__": + main() diff --git a/ci/spack.yaml b/ci/spack.yaml index 1065495ca..2f98ed45e 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -1,7 +1,7 @@ # ###################################################################### # In this environment, we specify that some packages be built with the -# native compiler (currently GCC 13 on Ubuntu 24.04), and others with -# the compiler required to build Phlex (currently GCC 15). +# native compiler (currently GCC ${PHLEX_NATIVE_GCC_VERSION}), and others with +# the compiler required to build Phlex (currently GCC ${PHLEX_GCC_VERSION}). # ###################################################################### spack: @@ -16,7 +16,7 @@ spack: - py-pytest-cov # Needed for Python coverage reports - py-pyyaml # Needed by run-clang-tidy --export-fixes to merge per-TU fix files - | - llvm@22.1.8 +zstd +llvm_dylib +link_llvm_dylib ~lldb targets=x86 + llvm@${PHLEX_LLVM_VERSION} +zstd +llvm_dylib +link_llvm_dylib ~lldb targets=${PHLEX_LLVM_TARGET} view: default: @@ -38,20 +38,19 @@ spack: require: - "~qtgui" - "@4:" - - "target=x86_64_v3" + - "target=${PHLEX_SPACK_TARGET}" llvm: require: - - "target=x86_64_v3" - - "%c,cxx=gcc@15" + - "targets=${PHLEX_LLVM_TARGET}" + - "%c,cxx=gcc@${PHLEX_GCC_VERSION}" phlex: require: - "@develop" - "+form" - "cxxstd=23" - - "%cxx=gcc@15" - - "^intel-tbb@2023.0.0:" + - "%cxx=gcc@${PHLEX_GCC_VERSION}" py-cython: require: @@ -74,13 +73,15 @@ spack: require: - "@3.12" - # root and its dependents (phlex) must be built with gcc@15 so - # they share a single ABI. root's own dependencies (e.g. zlib, - # python, xrootd) inherit the gcc@13 site default instead. + # ROOT and its dependents (e.g. phlex) must be built with + # ${PHLEX_GCC_VERSION} so they share a single ABI. ROOT's own + # dependencies (e.g. zlib, python, xrootd) inherit the + # gcc@${PHLEX_NATIVE_GCC_VERSION} native compiler dependency. + # # ROOT should be built with the same version of the C++ standard # as will be used by Phlex (C++23 for now). root: require: - "~x" # No graphics libraries required - "cxxstd=23" - - "%c,cxx=gcc@15" + - "%c,cxx=gcc@${PHLEX_GCC_VERSION}" diff --git a/codecov.yml b/codecov.yml index aa797bcf2..d55877670 100644 --- a/codecov.yml +++ b/codecov.yml @@ -9,37 +9,47 @@ coverage: status: project: default: - # Restrict to unittests flag so scripts coverage does not dilute - # the project-wide 80% threshold. flags: - - unittests + - amd64 + paths: + - phlex/** + - form/** + - plugins/** target: 80% threshold: 2% if_ci_failed: error patch: default: - # Restrict to unittests flag for the same reason. flags: - - unittests + - amd64 + paths: + - phlex/** + - form/** + - plugins/** target: 80% threshold: 5% +component_management: + individual_components: + - component_id: framework + name: Framework + paths: + - phlex/** + - form/** + - plugins/** + - component_id: scripts + name: Scripts + paths: + - scripts/** + - ci/** + - .devcontainer/configure-host.py + flags: - unittests: - # Coverage from the main C++ and Python test suite. - paths: - - "!scripts/" - carryforward: true - scripts: - # Coverage for scripts/ utility modules. No status thresholds are set - # yet; the flag is tracked and visible in the CodeCov UI but does not - # gate CI. - paths: - - scripts/ - carryforward: true + amd64: + arm64: comment: - layout: "header, diff, flags, files, footer" + layout: "header, diff, flags, components, files, footer" behavior: default require_changes: false require_base: false @@ -53,7 +63,6 @@ ignore: - "build-*/**" # Ignore build directories (generic 2/3) - "*-build/**" # Ignore build directories (generic 3/3) - ".coverage-generated/**" # Ignore normalized symlink tree - - "ci/**" # Ignore CI scripts - "docs/**" # Ignore documentation - "examples/**" # Ignore example code diff --git a/scripts/generate-image b/scripts/generate-image new file mode 100755 index 000000000..7f33b8f1e --- /dev/null +++ b/scripts/generate-image @@ -0,0 +1,98 @@ +#!/bin/bash + +if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then + printf '%s\n' 'ERROR: generate-image must be executed, not sourced' >&2 + return 1 +fi + +# Build the Phlex CI or development podman image using ci/Dockerfile. +# See the Podman instructions at the top of ci/Dockerfile for details. + +set -eu + +usage() { + echo "Usage: $(basename "$0") ci|dev" >&2 + exit 2 +} + +if [ "$#" -ne 1 ]; then + usage +fi + +case "$1" in + ci | dev) PHLEX_PODMAN_TARGET="$1" ;; + *) + echo "error: image identifier must be 'ci' or 'dev', got '$1'" >&2 + usage + ;; +esac + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_SOURCE="$(dirname "$SCRIPT_DIR")" + +: "${GPG_KEY_ID:=}" +: "${PHLEX_TAG_EXTRA:=}" +: "${PHLEX_DEFAULT_COMPILER:=gcc}" +: "${PHLEX_GCC_VERSION:=15.3}" +: "${PHLEX_LLVM_TARGET:=auto}" +: "${PHLEX_LLVM_VERSION:=22.1.8}" +: "${PHLEX_SPACK_CACHE:=}" + +host_arch=$(uname -m) +case "$host_arch" in + amd64 | x86_64) host_target=x86_64_v3 ;; + arm64 | aarch64) host_target=aarch64 ;; + *) + echo "error: unsupported platform '$host_arch'" >&2 + exit 1 + ;; +esac + +case "${PHLEX_SPACK_TARGET:-auto}" in + auto) PHLEX_SPACK_TARGET="$host_target" ;; + amd64 | x86_64_v3) PHLEX_SPACK_TARGET=x86_64_v3 ;; + arm64 | aarch64) PHLEX_SPACK_TARGET=aarch64 ;; + *) + echo "error: unsupported PHLEX_SPACK_TARGET '$PHLEX_SPACK_TARGET'" >&2 + exit 1 + ;; +esac +if [ "$PHLEX_SPACK_TARGET" != "$host_target" ]; then + echo "error: host architecture '$host_arch' does not match PHLEX_SPACK_TARGET='$PHLEX_SPACK_TARGET'; images must be built natively" >&2 + exit 1 +fi + +: "${PHLEX_TAG_BASE:=phlex-${PHLEX_PODMAN_TARGET}}" +: "${PHLEX_TAG_TAG:=$(date +%F)${PHLEX_TAG_EXTRA}}" +: "${PHLEX_TAG_REMOTE:=ghcr.io/framework-r-d/}" + +PHLEX_TAG_LOCAL="${PHLEX_TAG_BASE}:${PHLEX_TAG_TAG}-${PHLEX_SPACK_TARGET}" + +args=(podman build --format docker) +have_cache=0 + +if [ -n "$PHLEX_SPACK_CACHE" ]; then + have_cache=1 + args+=(-v "${PHLEX_SPACK_CACHE}:/build-cache:Z") +else + echo "info: no local cache configured" +fi +if [ -d "${HOME}/.gnupg" ]; then + args+=(-v "${HOME}/.gnupg:/run/gpg:Z") +elif (( have_cache )); then + echo "info: packages written to cache will be unsigned" +fi +args+=(--target "${PHLEX_PODMAN_TARGET}" --tag "${PHLEX_TAG_LOCAL}") +if [ -n "$GPG_KEY_ID" ]; then + args+=(--build-arg "GPG_KEY_ID=${GPG_KEY_ID}") +fi +args+=(--build-arg "PHLEX_SPACK_TARGET=${PHLEX_SPACK_TARGET}") +args+=(--build-arg "PHLEX_LLVM_TARGET=${PHLEX_LLVM_TARGET}") +args+=(--build-arg "PHLEX_GCC_VERSION=${PHLEX_GCC_VERSION}") +args+=(--build-arg "PHLEX_LLVM_VERSION=${PHLEX_LLVM_VERSION}") +args+=(--build-arg "PHLEX_DEFAULT_COMPILER=${PHLEX_DEFAULT_COMPILER}") + +echo "${args[*]}" + +cd "${PROJECT_SOURCE}/ci" +exec "${args[@]}" . diff --git a/scripts/setup-env.sh b/scripts/setup-env.sh index adc6780cd..b51f621da 100644 --- a/scripts/setup-env.sh +++ b/scripts/setup-env.sh @@ -12,6 +12,13 @@ # Usage: source scripts/setup-env.sh # . scripts/setup-env.sh +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + echo "Error: This script should be sourced, not executed directly" >&2 + echo "Usage: source scripts/setup-env.sh" >&2 + echo " . scripts/setup-env.sh" >&2 + exit 1 +fi + # Colors for output if [[ -t 1 ]]; then RED='\033[0;31m' @@ -29,54 +36,37 @@ fi # Output functions log() { - echo -e "${BLUE}[Phlex Setup]${NC} $1" + printf '%b[Phlex Setup]%b %s\n' "${BLUE-}" "${NC-}" "$*" } success() { - echo -e "${GREEN}[Phlex Setup]${NC} $1" + printf '%b[Phlex Setup]%b %s\n' "${GREEN-}" "${NC-}" "$*" } error() { - echo -e "${RED}[Phlex Setup]${NC} $1" >&2 + printf '%b[Phlex Setup]%b %s\n' "${RED-}" "${NC-}" "$*" >&2 } warn() { - echo -e "${YELLOW}[Phlex Setup]${NC} $1" -} - -# Function to handle errors and return instead of exit -handle_error() { - local line_no=$1 - local error_code=$2 - error "Error on line $line_no: Command exited with status $error_code" - error "Failed to set up Phlex environment" - trap - ERR - return $error_code + printf '%b[Phlex Setup]%b %s\n' "${YELLOW-}" "${NC-}" "$*" } -# Ensure script is sourced, not executed -if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then - # Script is being sourced - set up error trap - trap 'handle_error ${LINENO} $?' ERR -else - # Script is being executed directly - echo "Error: This script should be sourced, not executed directly" >&2 - echo "Usage: source scripts/setup-env.sh" >&2 - echo " . scripts/setup-env.sh" >&2 - exit 1 -fi - log "Setting up Phlex build environment..." # Get the directory of this script -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_SOURCE="$(dirname "$SCRIPT_DIR")" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" || return 1 +PROJECT_SOURCE="$(dirname "$SCRIPT_DIR")" || return 1 # Detect workspace structure and build mode detect_build_mode() { # Check if we're in a multi-project structure # Indicators: parent directory has 'srcs' subdirectory containing our project - local potential_workspace="$(dirname "$(dirname "$PROJECT_SOURCE")")" + local potential_workspace + if [[ -z "${PROJECT_SOURCE:-}" ]]; then + error "PROJECT_SOURCE must be set before detecting build mode" + return 1 + fi + potential_workspace="$(dirname "$(dirname "$PROJECT_SOURCE")")" || return 1 if [[ -d "$potential_workspace/srcs" && -d "$potential_workspace/srcs/phlex" ]]; then # Multi-project mode @@ -89,7 +79,7 @@ detect_build_mode() { else # Standalone mode BUILD_MODE="standalone" - WORKSPACE_ROOT="$(dirname "$PROJECT_SOURCE")" + WORKSPACE_ROOT="$(dirname "$PROJECT_SOURCE")" || return 1 SOURCE_ROOT="$PROJECT_SOURCE" PHLEX_SOURCE_DIR="$PROJECT_SOURCE" PHLEX_BUILD_DIR="${PHLEX_BUILD_DIR:-$WORKSPACE_ROOT/build}" @@ -105,7 +95,7 @@ detect_build_mode() { # Detect CI/container environment detect_ci_environment() { - if [[ -n "${CI}" || -n "${GITHUB_ACTIONS}" ]]; then + if [[ -n "${CI:-}" || -n "${GITHUB_ACTIONS:-}" ]]; then log "Detected CI environment" return 0 fi @@ -121,32 +111,35 @@ detect_ci_environment() { # Setup Spack environment (optional - gracefully degrades) setup_spack_environment() { - # If SPACK_ROOT already set, Spack is already configured - if [[ -n "${SPACK_ROOT}" ]]; then - log "Spack already configured (SPACK_ROOT=${SPACK_ROOT})" - return 0 + local setup_path + local -a spack_setup_paths=() + + # SPACK_ROOT can be exported before its shell setup script is loaded. + if [[ -n "${SPACK_ROOT:-}" ]]; then + if command -v spack &> /dev/null; then + log "Spack already configured (SPACK_ROOT=${SPACK_ROOT})" + return 0 + fi + spack_setup_paths+=("${SPACK_ROOT}/share/spack/setup-env.sh") fi # Try to find and source Spack setup script - local spack_setup_paths=( - "/spack/share/spack/setup-env.sh" # Container - "${PHLEX_SPACK_ENV}/../../share/spack/setup-env.sh" # Custom env - "${HOME}/spack/share/spack/setup-env.sh" # User install + if [[ -n "${PHLEX_SPACK_ROOT:-}" ]]; then + spack_setup_paths+=("${PHLEX_SPACK_ROOT}/share/spack/setup-env.sh") + fi + spack_setup_paths+=("/spack/share/spack/setup-env.sh") + if [[ -n "${PHLEX_SPACK_ENV:-}" ]]; then + spack_setup_paths+=("${PHLEX_SPACK_ENV}/../../share/spack/setup-env.sh") + fi + if [[ -n "${HOME:-}" ]]; then + spack_setup_paths+=("${HOME}/spack/share/spack/setup-env.sh") + fi + spack_setup_paths+=( "/usr/local/spack/share/spack/setup-env.sh" # System install "/opt/spack/share/spack/setup-env.sh" # Alt system "/cvmfs/fermilab.opensciencegrid.org/products/spack/current/share/spack/setup-env.sh" # FNAL OSG ) - # Add user-specified path if provided - if [[ -n "${SPACK_ROOT}" ]]; then - # Spack already configured, re-source to ensure functions are loaded - if [[ -f "${SPACK_ROOT}/share/spack/setup-env.sh" ]]; then - spack_setup_paths=("${SPACK_ROOT}/share/spack/setup-env.sh" "${spack_setup_paths[@]}") - fi - elif [[ -n "${PHLEX_SPACK_ROOT}" ]]; then - spack_setup_paths=("${PHLEX_SPACK_ROOT}/share/spack/setup-env.sh" "${spack_setup_paths[@]}") - fi - for setup_path in "${spack_setup_paths[@]}"; do if [[ -f "$setup_path" ]]; then log "Loading Spack from: $setup_path" @@ -176,12 +169,12 @@ activate_spack_environment() { local env_status env_status="$(spack env status 2>/dev/null || echo "No active environment")" if [[ "$env_status" != *"No active"* ]]; then - log "Spack environment already active: $(spack env status)" + log "Spack environment already active: $env_status" return 0 fi # Try to use Spack MPD if available (multi-project workflows) - if [[ "$BUILD_MODE" == "multi-project" ]] && spack commands 2>/dev/null | grep -q "^mpd$"; then + if [[ "${BUILD_MODE:-}" == "multi-project" ]] && spack commands 2>/dev/null | grep "^mpd$" >/dev/null; then log "Attempting to select project with spack mpd..." if spack mpd select . 2>/dev/null; then log "Successfully selected project with spack mpd" @@ -191,7 +184,7 @@ activate_spack_environment() { fi # Try user-specified environment - if [[ -n "${PHLEX_SPACK_ENV}" ]]; then + if [[ -n "${PHLEX_SPACK_ENV:-}" ]]; then log "Activating Spack environment: ${PHLEX_SPACK_ENV}" if spack env activate "${PHLEX_SPACK_ENV}" 2>/dev/null; then success "Activated Spack environment: ${PHLEX_SPACK_ENV}" @@ -202,8 +195,8 @@ activate_spack_environment() { fi # Try to activate local environment if it exists - local local_env="${WORKSPACE_ROOT}/local" - if [[ -d "$local_env" && -f "$local_env/spack.yaml" ]]; then + local local_env="${WORKSPACE_ROOT:-}/local" + if [[ -n "${WORKSPACE_ROOT:-}" && -d "$local_env" && -f "$local_env/spack.yaml" ]]; then log "Activating local Spack environment: ${local_env}" if spack env activate "${local_env}" 2>/dev/null; then success "Activated local Spack environment" @@ -237,10 +230,20 @@ check_build_tools() { # Check CMake version local cmake_version - cmake_version=$(cmake --version | head -n1 | sed 's/cmake version //') + cmake_version=$(cmake --version) || { + error "Unable to determine CMake version" + return 1 + } + cmake_version=${cmake_version%%$'\n'*} + cmake_version=${cmake_version#cmake version } local major minor - major=$(echo "${cmake_version}" | cut -d. -f1) - minor=$(echo "${cmake_version}" | cut -d. -f2) + if [[ "$cmake_version" =~ ^([0-9]+)\.([0-9]+) ]]; then + major=${BASH_REMATCH[1]} + minor=${BASH_REMATCH[2]} + else + error "Unrecognized CMake version: $cmake_version" + return 1 + fi if [[ ${major} -lt 3 ]] || [[ ${major} -eq 3 && ${minor} -lt 24 ]]; then warn "CMake version ${cmake_version} may be too old (minimum 3.24 recommended)" @@ -314,7 +317,7 @@ check_build_tools() { # Main setup flow main_setup() { # Detect build mode and set paths - detect_build_mode + detect_build_mode || return 1 # Ensure build directory exists if ! mkdir -p "${PHLEX_BUILD_DIR}" 2>/dev/null; then @@ -323,10 +326,10 @@ main_setup() { fi # Try to setup Spack (optional - will warn if not found) - setup_spack_environment + setup_spack_environment || : # Try to activate Spack environment (optional) - activate_spack_environment + activate_spack_environment || : # Set CMake-specific variables for consistent builds export CMAKE_EXPORT_COMPILE_COMMANDS=ON @@ -335,7 +338,6 @@ main_setup() { # Check for required tools (fails if critical tools missing) if ! check_build_tools; then error "Environment setup incomplete - missing critical build tools" - trap - ERR return 1 fi @@ -346,7 +348,7 @@ main_setup() { log "Workspace root: ${WORKSPACE_ROOT}" log "Source directory: ${PHLEX_SOURCE_DIR}" log "Build directory: ${PHLEX_BUILD_DIR}" - if [[ -n "${CMAKE_PREFIX_PATH}" ]]; then + if [[ -n "${CMAKE_PREFIX_PATH:-}" ]]; then log "CMAKE_PREFIX_PATH: ${CMAKE_PREFIX_PATH}" fi if command -v spack &> /dev/null; then @@ -374,17 +376,15 @@ main_setup() { echo "" fi - # Clean up and return success - trap - ERR return 0 } -# Run main setup and capture result -main_setup -setup_result=$? - -# Clean up error trap -trap - ERR +# Handle failure explicitly without replacing the sourcing shell's ERR trap. +if main_setup; then + setup_result=0 +else + setup_result=$? +fi # Return the result return $setup_result diff --git a/scripts/test/CMakeLists.txt b/scripts/test/CMakeLists.txt index 7d7b98457..867b4964c 100644 --- a/scripts/test/CMakeLists.txt +++ b/scripts/test/CMakeLists.txt @@ -33,20 +33,20 @@ add_test( WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} ) -# Coverage target for the scripts/ utility modules. Mirrors the approach used -# by test/python/CMakeLists.txt for the coverage-python target. +# Coverage target for Python utilities in scripts/, ci/, and .devcontainer/. if(HAS_SCRIPTS_PYTEST_COV AND ENABLE_COVERAGE) add_custom_target( coverage-scripts COMMAND ${Python_EXECUTABLE} -m pytest ${CMAKE_CURRENT_SOURCE_DIR} - --cov=${PROJECT_SOURCE_DIR}/scripts --omit=${CMAKE_CURRENT_SOURCE_DIR}/* + --cov=${PROJECT_SOURCE_DIR}/scripts --cov=${PROJECT_SOURCE_DIR}/ci + --cov=${PROJECT_SOURCE_DIR}/.devcontainer --omit=${CMAKE_CURRENT_SOURCE_DIR}/* --cov-report=term-missing --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-scripts.xml WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - COMMENT "Running scripts/ Python coverage report" + COMMENT "Running scripts, CI, and devcontainer Python coverage report" VERBATIM ) - message(STATUS "Added 'coverage-scripts' target for scripts/ utility modules.") + message(STATUS "Added 'coverage-scripts' target for repository Python utilities.") elseif(ENABLE_COVERAGE AND NOT HAS_SCRIPTS_PYTEST_COV) message(WARNING "ENABLE_COVERAGE is ON but pytest-cov not found; scripts coverage disabled.") endif() diff --git a/scripts/test/test_devcontainer_host.py b/scripts/test/test_devcontainer_host.py new file mode 100644 index 000000000..18e00decb --- /dev/null +++ b/scripts/test/test_devcontainer_host.py @@ -0,0 +1,1011 @@ +"""Regression tests for native devcontainer selection and private auth seeding.""" + +from __future__ import annotations + +import json +import os +import runpy +import shutil +import subprocess +from pathlib import Path +from unittest.mock import patch + +import pytest + +DEVCONTAINER = Path(__file__).resolve().parents[2] / ".devcontainer" +CONFIGURE_HOST = runpy.run_path(str(DEVCONTAINER / "configure-host.py")) +HOST_SERVICE = CONFIGURE_HOST["host_service"] +HOST_MAIN = CONFIGURE_HOST["main"] + + +@pytest.fixture(autouse=True) +def clean_image_override(monkeypatch): + """Do not let the developer's selected image influence fixtures.""" + monkeypatch.delenv("PHLEX_DEV_IMAGE", raising=False) + monkeypatch.delenv("PHLEX_ENABLE_NESTED_PODMAN", raising=False) + + +def test_darwin_native_defaults(tmp_path): + """Use pasta and a local arm64 image without engine access or registry pulls.""" + with patch( + "subprocess.check_output", + side_effect=["arm64 true unix:///run/user/550/podman/podman.sock\n", "linux/arm64\n"], + ) as command: + service = HOST_SERVICE("Darwin", tmp_path) + assert service["network_mode"] == "pasta" + assert service["platform"] == "linux/arm64" + assert service["volumes"] == [] + assert "security_opt" not in service + assert "environment" not in service + assert service["build"]["args"]["PHLEX_DEV_IMAGE"] == "localhost/phlex-dev:latest" + assert command.call_args_list[1].args[0] == [ + "podman", + "image", + "inspect", + "--format", + "{{.Os}}/{{.Architecture}}", + "localhost/phlex-dev:latest", + ] + + +def test_darwin_nested_podman_is_explicit(tmp_path, monkeypatch): + """Only explicit opt-in grants engine access and disables SELinux labeling.""" + monkeypatch.setenv("PHLEX_ENABLE_NESTED_PODMAN", "1") + with patch( + "subprocess.check_output", + side_effect=["arm64 true /run/user/550/podman/podman.sock", "linux/arm64"], + ): + service = HOST_SERVICE("Darwin", tmp_path) + assert service["volumes"] == ["/run/user/550/podman/podman.sock:/tmp/podman.sock:ro"] + assert service["security_opt"] == ["label=disable"] + assert service["environment"]["DOCKER_HOST"] == "unix:///tmp/podman.sock" + + +def test_darwin_explicit_native_image(tmp_path, monkeypatch): + """An override must be inspected locally before it is selected for macOS.""" + monkeypatch.setenv("PHLEX_DEV_IMAGE", "localhost/phlex-dev:custom") + with patch( + "subprocess.check_output", + side_effect=["arm64 true /run/user/550/podman/podman.sock", "linux/arm64"], + ) as command: + service = HOST_SERVICE("Darwin", tmp_path) + assert service["build"]["args"]["PHLEX_DEV_IMAGE"] == "localhost/phlex-dev:custom" + assert command.call_args.args[0][-1] == "localhost/phlex-dev:custom" + + +@pytest.mark.parametrize( + "info", + [ + "amd64 true unix:///run/user/550/podman/podman.sock", + "arm64 false unix:///run/podman/podman.sock", + "arm64 true unix:///Users/someone/podman.sock", + "invalid", + ], +) +def test_darwin_rejects_wrong_backend(tmp_path, info): + """Reject emulated, rootful, malformed and host-filesystem socket selections.""" + with patch("subprocess.check_output", return_value=info) as command: + with pytest.raises(ValueError): + HOST_SERVICE("Darwin", tmp_path) + assert command.call_count == 1 + + +@pytest.mark.parametrize("image_platform", ["linux/amd64", "windows/arm64", "", "arm64"]) +def test_darwin_rejects_wrong_image_platform(tmp_path, image_platform): + """Reject both incompatible OS/architecture and malformed inspection results.""" + with patch( + "subprocess.check_output", + side_effect=["arm64 true /run/user/550/podman/podman.sock", image_platform], + ): + with pytest.raises(ValueError, match="does not resolve to a local linux/arm64 image"): + HOST_SERVICE("Darwin", tmp_path) + + +@pytest.mark.parametrize( + "image", + [ + "ghcr.io/framework-r-d/phlex-dev:latest", + "ghcr.io/framework-r-d/phlex-dev@sha256:" + "a" * 64, + ], +) +def test_darwin_multiarch_reference_uses_local_native_member(tmp_path, monkeypatch, image): + """A registry tag or index digest works when Podman resolves its native member locally.""" + monkeypatch.setenv("PHLEX_DEV_IMAGE", image) + with patch( + "subprocess.check_output", + side_effect=["arm64 true /run/user/550/podman/podman.sock", "linux/arm64"], + ) as command: + service = HOST_SERVICE("Darwin", tmp_path) + assert service["build"]["args"]["PHLEX_DEV_IMAGE"] == image + assert service["platform"] == "linux/arm64" + assert command.call_count == 2 + assert command.call_args.args[0][-1] == image + + +@pytest.mark.parametrize("image", [None, "ghcr.io/framework-r-d/phlex-dev:multiarch"]) +def test_missing_image_does_not_fall_back(tmp_path, monkeypatch, image): + """An absent native image must not cause an automatic registry pull.""" + if image is not None: + monkeypatch.setenv("PHLEX_DEV_IMAGE", image) + with patch( + "subprocess.check_output", + side_effect=[ + "arm64 true /run/user/550/podman/podman.sock", + subprocess.CalledProcessError(125, "podman image inspect"), + ], + ) as command: + with pytest.raises(subprocess.CalledProcessError): + HOST_SERVICE("Darwin", tmp_path) + assert command.call_count == 2 + + +def test_linux_keeps_base_defaults(tmp_path): + """Linux requires neither a Podman VM nor an existing credentials file.""" + with patch("subprocess.check_output") as command: + assert HOST_SERVICE("Linux", tmp_path) == {"volumes": []} + command.assert_not_called() + + +def test_linux_mounts_existing_podman_proxy(tmp_path): + """An available VM proxy socket is mounted and exported for container clients.""" + proxy_socket = tmp_path / ".podman-proxy/podman.sock" + proxy_socket.parent.mkdir(parents=True) + proxy_socket.touch() + + service = HOST_SERVICE("Linux", tmp_path) + + assert service["volumes"] == [f"{proxy_socket}:/tmp/podman.sock:Z"] + assert service["environment"] == { + "DOCKER_HOST": "unix:///tmp/podman.sock", + "CONTAINER_HOST": "unix:///tmp/podman.sock", + } + + +def test_explicit_image_and_auth_destination(tmp_path, monkeypatch): + """Explicit images work and host auth never masks the writable private auth file.""" + monkeypatch.setenv("PHLEX_DEV_IMAGE", "localhost/phlex-dev:custom") + auth = tmp_path / ".local/share/kilo/auth.json" + auth.parent.mkdir(parents=True) + auth.write_text("{}\n") + service = HOST_SERVICE("Linux", tmp_path) + assert service["build"]["args"]["PHLEX_DEV_IMAGE"] == "localhost/phlex-dev:custom" + assert service["volumes"] == [f"{auth}:/run/phlex-host-auth.json:ro,Z"] + assert auth.read_text() == "{}\n" + + +def _patch_host_main(monkeypatch, target, home): + main_globals = HOST_MAIN.__globals__ + monkeypatch.setitem(main_globals, "__file__", str(target.with_name("configure-host.py"))) + monkeypatch.setattr(main_globals["platform"], "system", lambda: "Linux") + monkeypatch.setattr(main_globals["Path"], "home", staticmethod(lambda: home)) + + +def test_main_atomically_writes_compose_override(tmp_path, monkeypatch): + """The generated override replaces stale content and removes its same-dir temp file.""" + target = tmp_path / "checkout with spaces" / "docker-compose.host.yml" + target.parent.mkdir() + target.write_text("stale override\n") + home = tmp_path / "home with spaces" + auth = home / ".local/share/kilo/auth.json" + auth.parent.mkdir(parents=True) + auth.write_text("{}\n") + _patch_host_main(monkeypatch, target, home) + + with patch.object(CONFIGURE_HOST["os"], "replace", wraps=os.replace) as replace: + HOST_MAIN() + + replace.assert_called_once() + temporary = Path(replace.call_args.args[0]) + assert temporary.parent == target.parent + assert replace.call_args.args[1] == target + assert not temporary.exists() + assert target.read_text().endswith("\n") + service = json.loads(target.read_text())["services"]["phlex-dev"] + assert service["volumes"] == [f"{auth}:/run/phlex-host-auth.json:ro,Z"] + + +def test_main_preserves_override_and_cleans_temp_on_replace_failure(tmp_path, monkeypatch): + """A failed atomic replacement leaves the prior override intact and cleans up.""" + target = tmp_path / "docker-compose.host.yml" + target.write_text("previous override\n") + _patch_host_main(monkeypatch, target, tmp_path / "home") + + def fail_replace(source, destination): + raise OSError("simulated replace failure") + + monkeypatch.setattr(CONFIGURE_HOST["os"], "replace", fail_replace) + with pytest.raises(OSError, match="simulated replace failure"): + HOST_MAIN() + + assert target.read_text() == "previous override\n" + assert list(target.parent.glob(".compose-host-*")) == [] + + +def test_auth_seed_preserves_other_providers(tmp_path, monkeypatch): + """Seeding an environment key must preserve existing unrelated provider credentials.""" + auth = tmp_path / "auth.json" + auth.write_text('{"other-provider": {"type": "api", "key": "fixture"}}\n') + monkeypatch.setenv("KILO_API_KEY", "fixture-new-key") + script = (DEVCONTAINER / "post-create.sh").read_text() + seed = script.split("python3 - <<'PY'\n", 1)[1].split("\nPY", 1)[0] + with patch("pathlib.Path", return_value=auth): + exec(compile(seed, "post-create.sh auth seed", "exec"), {}) + result = json.loads(auth.read_text()) + assert result["other-provider"]["key"] == "fixture" + assert result["fnal-azure"]["key"] == "fixture-new-key" + assert result["fnal-ow"]["key"] == "fixture-new-key" + assert auth.stat().st_mode & 0o777 == 0o600 + + +@pytest.mark.parametrize("compiler", ["gcc", "clang"]) +@pytest.mark.parametrize("version,native", [("15.3", "13.3.0"), ("16.2", "14.3"), (None, None)]) +def test_compiler_uses_build_versions(tmp_path, monkeypatch, compiler, version, native): + """Both compiler modes use build-provided versions, not fixed GCC major versions.""" + for key, value in [("PHLEX_GCC_VERSION", version), ("PHLEX_NATIVE_GCC_VERSION", native)]: + if value is None: + monkeypatch.delenv(key, raising=False) + else: + monkeypatch.setenv(key, value) + gcc = tmp_path / "bin/gcc" + gcc.parent.mkdir() + gcc.write_text("#!/bin/sh\nprintf '%s\\n' /fixture/lib/gcc/aarch64/15.3.0/libgcc.a\n") + gcc.chmod(0o700) + entrypoint = (DEVCONTAINER.parent / "ci/entrypoint.sh").read_text() + start = entrypoint.index("gcc_path=$(spack -E location") + gcc_spec = "gcc" + (f"@{version}" if version else "") + native_spec = "%c,cxx=gcc" + (f"@{native}" if native else "") + script = f""" +set -eu +spack() {{ + [[ "$1" = -E && "$2" = location && "$3" = -i ]] || return 1 + [[ "$4" = '{gcc_spec}' && "$5" = '{native_spec}' ]] || return 1 + printf '%s\\n' '{tmp_path}' +}} +PHLEX_DEFAULT_COMPILER={compiler} +unset CXXFLAGS +{entrypoint[start:]} +printf '%s\\n' "$CXX" "${{CXXFLAGS:-}}" "${{PATH%%:*}}" +""" + result = subprocess.run(["bash", "-c", script], capture_output=True, text=True, check=True) + lines = result.stdout.splitlines() + if compiler == "clang": + assert lines[:2] == ["clang++", "--gcc-install-dir=/fixture/lib/gcc/aarch64/15.3.0"] + else: + assert lines == ["g++", "", str(tmp_path / "bin")] + + +def test_kilo_config_discovery_uses_mount(): + """Production discovery reads the config mount rather than the relay-state mount.""" + script = f""" +source '{DEVCONTAINER}/kilo-env.sh' +[() {{ + if [[ "$1" = -f ]]; then + [[ "$2" = /root/.config/kilo/kilo.jsonc ]] + else + builtin [ "$@" + fi +}} +kilo_env_find_config_path /run/phlex-host-relays +""" + result = subprocess.run(["bash", "-c", script], capture_output=True, text=True, check=True) + assert result.stdout.strip() == "/root/.config/kilo/kilo.jsonc" + + +@pytest.mark.parametrize( + "target,llvm,compiler,message", + [ + ("invalid", "auto", "gcc", "PHLEX_SPACK_TARGET must be"), + ("aarch64", "x86", "gcc", "Host/target mismatch"), + ("aarch64", "aarch64", "invalid", "PHLEX_DEFAULT_COMPILER must be"), + ], +) +@pytest.mark.parametrize("options", ["", "-e", "-u", "-eu"]) +def test_entrypoint_validation_preserves_sourcing_shell( + tmp_path, target, llvm, compiler, message, options +): + """Invalid configuration fails without closing a shell that sources the entrypoint.""" + entrypoint = (DEVCONTAINER.parent / "ci/entrypoint.sh").read_text() + validation = entrypoint.split(". /spack/share/spack/setup-env.sh", 1)[0] + validation = validation.replace( + "if [ -f /etc/profile.d/phlex-targets.sh ]; then", "if false; then" + ) + fixture = tmp_path / "entrypoint.sh" + fixture.write_text(validation.rstrip() + "\n") + env = { + **os.environ, + "PHLEX_SPACK_TARGET": target, + "PHLEX_LLVM_TARGET": llvm, + "PHLEX_DEFAULT_COMPILER": compiler, + } + command = [ + "bash", + "-c", + f'set {options or "+e"}; if source "$1"; then status=0; else status=$?; fi; ' + 'printf "survived\\n"; exit "$status"', + "bash", + str(fixture), + ] + result = subprocess.run(command, env=env, capture_output=True, text=True) + assert result.returncode == 1 + assert message in result.stderr + assert "survived" in result.stdout + + +@pytest.mark.parametrize( + "mode", ["resolve_existing_numeric_relay_state", "cleanup_owned_numeric_relay"] +) +def test_relay_pid_reuse_drops_record_without_signaling(tmp_path, mode): + """A live unrelated PID must neither block startup nor be killed during cleanup.""" + definitions = ( + (DEVCONTAINER / "ensure-repos.sh").read_text().split("# --- Main execution ---", 1)[0] + ) + script = ( + definitions + + f""" +printf '%s\\n' "$$" > "$(relay_pid_file 25114)" +terminate_pid_gracefully() {{ exit 99; }} +{mode} 25114 25115 || exit 1 +[ ! -f "$(relay_pid_file 25114)" ] || exit 2 +kill -0 "$$" +""" + ) + result = subprocess.run( + ["bash"], + input=script, + text=True, + capture_output=True, + env={**os.environ, "PHLEX_RELAY_STATE_DIR": str(tmp_path)}, + ) + assert result.returncode == 0, result.stderr + assert "discarding stale PID file" in result.stderr + + +@pytest.mark.parametrize("script", ["kilo-env.sh", "ensure-repos.sh"]) +def test_production_scripts_work_without_fixture_helpers(tmp_path, script): + """Copying only production code supports normal startup/cleanup but rejects self-testing.""" + fixture = tmp_path / script + fixture.write_text((DEVCONTAINER / script).read_text()) + env = {**os.environ, "PHLEX_RELAY_STATE_DIR": str(tmp_path / "relay-state")} + if script == "kilo-env.sh": + normal = ["bash", "-eu", "-c", 'source "$1"', "bash", str(fixture)] + optional = ["bash", "-eu", str(fixture), "--self-test"] + else: + normal = ["bash", str(fixture), "--cleanup-owned"] + optional = ["bash", str(fixture), "--self-test", str(tmp_path / "test-root")] + result = subprocess.run(normal, env=env, text=True, capture_output=True) + assert result.returncode == 0, result.stderr + result = subprocess.run(optional, env=env, text=True, capture_output=True) + assert result.returncode != 0 + assert "self-test" in result.stderr or "self_test" in result.stderr + + +@pytest.mark.parametrize("signum,status", [("HUP", 129), ("INT", 130), ("TERM", 143)]) +def test_relay_self_test_signal_stops_and_cleans_once(tmp_path, signum, status): + """Signals stop the self-test immediately and do not repeat its EXIT cleanup.""" + definitions = ( + (DEVCONTAINER / "ensure-repos.sh").read_text().split("# --- Main execution ---", 1)[0] + ) + cleanup = tmp_path / "cleanup" + script = ( + definitions + + f""" +source '{DEVCONTAINER}/ensure-repos-self-test.sh' +mode_cleanup_owned() {{ printf 'cleanup\\n' >> '{cleanup}'; }} +start_python_loopback_listener() {{ :; }} +wait_for_source_listener_ready() {{ kill -s {signum} "$$"; }} +mode_self_test_loaded +printf 'unexpected continuation\\n' +""" + ) + result = subprocess.run( + ["bash"], + input=script, + text=True, + capture_output=True, + timeout=10, + env={**os.environ, "PHLEX_RELAY_STATE_DIR": str(tmp_path / "state")}, + ) + assert result.returncode == status, result.stderr + assert "unexpected continuation" not in result.stdout + assert "Self-test: ready named services" not in result.stdout + assert cleanup.read_text() == "cleanup\n" + + +@pytest.mark.parametrize( + "host,target,expected", + [ + ("x86_64", None, "x86_64_v3"), + ("amd64", "amd64", "x86_64_v3"), + ("x86_64", "x86_64_v3", "x86_64_v3"), + ("arm64", "auto", "aarch64"), + ("aarch64", "arm64", "aarch64"), + ("arm64", "aarch64", "aarch64"), + ("x86_64", "aarch64", None), + ("arm64", "x86_64_v3", None), + ("arm64", "invalid", None), + ("unknown", None, None), + ], +) +def test_generate_image_native_target_and_tag(tmp_path, host, target, expected): + """Build tags and Spack targets are normalized and incompatible hosts fail early.""" + script = DEVCONTAINER.parent / "scripts/generate-image" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + podman = bin_dir / "podman" + podman.write_text('#!/bin/sh\nprintf "build invocation: %s\\n" "$*"\n') + podman.chmod(0o700) + command = f""" +uname() {{ printf '%s\\n' '{host}'; }} +export -f uname +bash "$1" dev +""" + env = { + key: value for key, value in os.environ.items() if not key.startswith(("PHLEX_", "GPG_")) + } + env.update(HOME=str(tmp_path), PHLEX_TAG_TAG="fixture", PATH=f"{bin_dir}:{os.environ['PATH']}") + if target is not None: + env["PHLEX_SPACK_TARGET"] = target + result = subprocess.run( + ["bash", "-c", command, "bash", str(script)], env=env, text=True, capture_output=True + ) + if expected is None: + assert result.returncode == 1 + assert "build invocation" not in result.stdout + assert "error:" in result.stderr + else: + assert result.returncode == 0, result.stderr + assert f"--tag phlex-dev:fixture-{expected}" in result.stdout + assert f"--build-arg PHLEX_SPACK_TARGET={expected}" in result.stdout + + +@pytest.mark.parametrize("template", ["packages.yaml", "spack.yaml"]) +def test_shared_spack_renderer_substitutes_build_inputs(tmp_path, template): + """Both Spack templates use the same renderer with build-selected versions and targets.""" + dockerfile = (DEVCONTAINER.parent / "ci/Dockerfile").read_text() + assert dockerfile.count("render_spack_template() {") == 1 + assert dockerfile.count(". /opt/render-spack-template.sh\n") == 2 + renderer = dockerfile.split("<<'RENDER_SPACK_TEMPLATE'\n", 1)[1].split( + "\nRENDER_SPACK_TEMPLATE", 1 + )[0] + values = { + "PHLEX_GCC_VERSION": "16.2", + "PHLEX_LLVM_TARGET": "aarch64", + "PHLEX_LLVM_VERSION": "23.1.0", + "PHLEX_NATIVE_GCC_VERSION": "14.3.0", + "PHLEX_SPACK_TARGET": "aarch64", + } + source = DEVCONTAINER.parent / "ci" / template + output = tmp_path / "rendered.yaml" + result = subprocess.run( + [ + "bash", + "-c", + renderer + '\nrender_spack_template "$1" "$2"', + "bash", + str(source), + str(output), + ], + env={**os.environ, **values}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + expected = source.read_text() + for key, value in values.items(): + expected = expected.replace("${" + key + "}", value) + assert output.read_text() == expected + assert "${PHLEX_" not in output.read_text() + + +@pytest.mark.parametrize("stage", ["targets", "setup", "activate", "location", "libgcc"]) +@pytest.mark.parametrize("options", ["", "-e", "-u", "-eu"]) +def test_entrypoint_dependency_failures_return_under_strict_options(tmp_path, stage, options): + """Dependency failures return predictably under either strict option or their combination.""" + targets = tmp_path / "targets.sh" + targets.write_text("return 1\n" if stage == "targets" else ":\n") + setup = tmp_path / "spack-setup.sh" + setup.write_text( + "return 1\n" + if stage == "setup" + else f"""spack() {{ + if [[ "$1" = env && "$2" = activate ]]; then + [[ '{stage}' != activate ]] + else + [[ '{stage}' != location ]] || return 1 + printf '%s\\n' '{tmp_path}' + fi +}} +""" + ) + gcc = tmp_path / "bin/gcc" + gcc.parent.mkdir() + gcc.write_text("#!/bin/sh\nexit 1\n") + gcc.chmod(0o700) + entrypoint = (DEVCONTAINER.parent / "ci/entrypoint.sh").read_text() + entrypoint = entrypoint.replace("/etc/profile.d/phlex-targets.sh", str(targets)).replace( + "/spack/share/spack/setup-env.sh", str(setup) + ) + fixture = tmp_path / "entrypoint.sh" + fixture.write_text(entrypoint) + env = { + **os.environ, + "PHLEX_SPACK_TARGET": "aarch64", + "PHLEX_LLVM_TARGET": "aarch64", + "PHLEX_DEFAULT_COMPILER": "clang", + } + command = [ + "bash", + "-c", + f'set {options or "+e"}; if source "$1"; then status=0; else status=$?; fi; ' + 'printf "survived\\n"; exit "$status"', + "bash", + str(fixture), + ] + result = subprocess.run(command, env=env, text=True, capture_output=True) + assert result.returncode == 1, result.stderr + assert "return: can only" not in result.stderr + assert "survived" in result.stdout + + +@pytest.mark.parametrize( + "script", + [ + ".devcontainer/ensure-repos.sh", + ".devcontainer/setup-repos.sh", + ".devcontainer/post-create.sh", + "scripts/generate-image", + ], +) +def test_execution_only_scripts_reject_sourcing_without_shell_changes(tmp_path, script): + """Accidental sourcing neither exits the caller nor replaces its options or traps.""" + command = """ +trap ':' EXIT HUP INT TERM ERR +before_options=$- +before_traps=$(trap -p) +source "$1" --self-test +status=$? +[[ "$status" = 1 && "$-" = "$before_options" && "$(trap -p)" = "$before_traps" ]] || exit 99 +printf 'survived\\n' +""" + result = subprocess.run( + ["bash", "-c", command, "bash", str(DEVCONTAINER.parent / script)], + cwd=tmp_path, + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr + assert result.stdout == "survived\n" + assert "must be executed, not sourced" in result.stderr + + +@pytest.mark.parametrize("status", [0, 1]) +def test_setup_env_preserves_caller_traps_and_options(tmp_path, status): + """Sourced setup preserves the caller's traps on both successful and failed setup.""" + script = (DEVCONTAINER.parent / "scripts/setup-env.sh").read_text() + definitions, main = script.split("# Handle failure explicitly", 1) + fixture = tmp_path / "setup-env.sh" + fixture.write_text( + definitions + + f""" +detect_build_mode() {{ PHLEX_BUILD_DIR='{tmp_path}'; }} +setup_spack_environment() {{ return 1; }} +activate_spack_environment() {{ return 1; }} +check_build_tools() {{ return {status}; }} +log() {{ :; }} +success() {{ :; }} +""" + + "# Handle failure explicitly" + + main + ) + command = """ +set -e +trap ':' EXIT HUP INT TERM ERR +before_options=$- +before_traps=$(trap -p) +if source "$1"; then status=0; else status=$?; fi +[[ "$status" = "$2" && "$-" = "$before_options" && "$(trap -p)" = "$before_traps" ]] || exit 99 +printf 'survived\\n' +""" + env = {key: value for key, value in os.environ.items() if not key.startswith("PHLEX_")} + result = subprocess.run( + ["bash", "-c", command, "bash", str(fixture), str(status)], + env=env, + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr + assert "survived" in result.stdout + + +def test_setup_env_rejects_execution(): + """The source-only setup script rejects execution before touching the environment.""" + result = subprocess.run( + ["bash", str(DEVCONTAINER.parent / "scripts/setup-env.sh")], + text=True, + capture_output=True, + ) + assert result.returncode == 1 + assert "should be sourced, not executed" in result.stderr + + +@pytest.mark.parametrize("sourced", [True, False]) +@pytest.mark.parametrize("self_test", [True, False]) +def test_kilo_env_exit_paths_preserve_callers(tmp_path, sourced, self_test): + """The dual-mode Kilo environment script returns when sourced and exits when executed.""" + script = DEVCONTAINER / "kilo-env.sh" + args = ["--self-test"] if self_test else [] + if sourced: + command_script = "; ".join( + ( + 'trap ":" EXIT HUP INT TERM ERR', + "before=$(trap -p)", + 'source "$1" "${@:2}"', + "status=$?", + '[[ "$(trap -p)" = "$before" ]] || exit 99', + 'printf "survived\\n"', + 'exit "$status"', + ) + ) + command = [ + "bash", + "-c", + command_script, + "bash", + str(script), + *args, + ] + else: + command = ["bash", str(script), *args] + env = {key: value for key, value in os.environ.items() if key != "PHLEX_RELAY_FIXTURE_DIR"} + result = subprocess.run(command, cwd=tmp_path, env=env, text=True, capture_output=True) + assert result.returncode == int(self_test), result.stderr + assert ("survived" in result.stdout) == sourced + + +@pytest.mark.parametrize("options", ["-e", "-u", "-eu"]) +def test_entrypoint_is_source_only(options): + """Executing the environment entrypoint fails before loading Spack or configuring exports.""" + result = subprocess.run( + ["bash", options, str(DEVCONTAINER.parent / "ci/entrypoint.sh")], + text=True, + capture_output=True, + ) + assert result.returncode == 1 + assert result.stderr.strip() == "ERROR: entrypoint.sh must be sourced, not executed" + + +@pytest.mark.parametrize("compiler", ["gcc", "clang"]) +@pytest.mark.parametrize("options", ["-e", "-u", "-eu"]) +def test_entrypoint_success_plain_source_under_strict_options(tmp_path, compiler, options): + """Plain sourcing succeeds with optional version/target variables absent and options intact.""" + targets = tmp_path / "targets.sh" + targets.write_text(":\n") + setup = tmp_path / "spack-setup.sh" + setup.write_text(f"""spack() {{ + if [[ "$1" = env ]]; then return 0; fi + printf '%s\\n' '{tmp_path}' +}} +""") + gcc = tmp_path / "bin/gcc" + gcc.parent.mkdir() + gcc.write_text("#!/bin/sh\nprintf '%s\\n' /fixture/lib/gcc/16.2/libgcc.a\n") + gcc.chmod(0o700) + fixture = tmp_path / "entrypoint.sh" + fixture.write_text( + (DEVCONTAINER.parent / "ci/entrypoint.sh") + .read_text() + .replace("/etc/profile.d/phlex-targets.sh", str(targets)) + .replace("/spack/share/spack/setup-env.sh", str(setup)) + ) + env = {key: value for key, value in os.environ.items() if not key.startswith("PHLEX_")} + env["PHLEX_DEFAULT_COMPILER"] = compiler + command = f""" +set {options} +trap ':' EXIT ERR HUP INT TERM +before_options=$- +before_traps=$(trap -p) +source "$1" +[[ "$-" = "$before_options" && "$(trap -p)" = "$before_traps" ]] +printf 'ready: %s\\n' "$CXX" +""" + result = subprocess.run( + ["bash", "-c", command, "bash", str(fixture)], env=env, text=True, capture_output=True + ) + assert result.returncode == 0, result.stderr + assert result.stdout == f"ready: {'g++' if compiler == 'gcc' else 'clang++'}\n" + + +@pytest.mark.parametrize("options", ["-e", "-u", "-eu"]) +@pytest.mark.parametrize("valid_cmake", [True, False]) +def test_setup_env_real_source_and_direct_functions_under_strict_options( + tmp_path, options, valid_cmake +): + """Exercise real setup and direct helpers without optional environment settings.""" + fixture = tmp_path / "setup-env.sh" + fixture.write_text((DEVCONTAINER.parent / "scripts/setup-env.sh").read_text()) + home = tmp_path / "home" + spack_setup = home / "spack/share/spack/setup-env.sh" + spack_setup.parent.mkdir(parents=True) + spack_setup.write_text(":\n") + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + for tool in ("cmake", "gcc", "g++", "ninja", "gcov", "lcov", "gcovr"): + executable = bin_dir / tool + executable.write_text( + "#!/bin/sh\n" + ("printf 'cmake version 3.31.0\\n'\n" if valid_cmake else "exit 1\n") + if tool == "cmake" + else "#!/bin/sh\nexit 0\n" + ) + executable.chmod(0o700) + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("PHLEX_", "SPACK_", "CMAKE_", "CI", "GITHUB_", "BUILD_")) + } + env.update( + HOME=str(home), PATH=f"{bin_dir}:/usr/bin:/bin", PHLEX_BUILD_DIR=str(tmp_path / "build") + ) + command = f""" +set {options} +before=$- +{'source "$1"' if valid_cmake else 'if source "$1"; then exit 99; fi'} +[[ "$-" = "$before" ]] +log +success +warn +error +if detect_ci_environment; then :; fi +setup_spack_environment +unset BUILD_MODE WORKSPACE_ROOT PHLEX_SPACK_ENV +spack() {{ if [[ "$1" = env && "$2" = status ]]; then return 1; fi; return 0; }} +activate_spack_environment +{"check_build_tools" if valid_cmake else "if check_build_tools; then exit 99; fi"} +printf 'survived\\n' +""" + result = subprocess.run( + ["bash", "-c", command, "bash", str(fixture)], env=env, text=True, capture_output=True + ) + assert result.returncode == 0, result.stderr + assert "unbound variable" not in result.stderr + assert "survived" in result.stdout + + +def test_setup_spack_environment_sources_spack_root_when_command_is_missing(tmp_path): + """An exported SPACK_ROOT still loads Spack's shell setup when its command is absent.""" + setup_script = tmp_path / "share/spack/setup-env.sh" + setup_script.parent.mkdir(parents=True) + setup_script.write_text('spack() { printf "fixture-spack\\n"; }\n') + + setup_env = (DEVCONTAINER.parent / "scripts/setup-env.sh").read_text() + definitions = setup_env.split("# Main setup flow", 1)[0] + fixture = tmp_path / "setup-env-functions.sh" + fixture.write_text(definitions) + command_script = "\n".join( + ( + 'source "$1"', + '[[ -n "${SPACK_ROOT:-}" ]]', + "if command -v spack >/dev/null 2>&1; then exit 99; fi", + "setup_spack_environment", + '[[ "$(spack)" = fixture-spack ]]', + ) + ) + env = { + key: value for key, value in os.environ.items() if not key.startswith(("PHLEX_", "SPACK_")) + } + env.update(SPACK_ROOT=str(tmp_path), PATH="/usr/bin:/bin") + result = subprocess.run( + ["bash", "-c", command_script, "bash", str(fixture)], + env=env, + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("options", ["-e", "-u", "-eu"]) +def test_kilo_env_plain_source_and_direct_functions_under_strict_options(tmp_path, options): + """Real Kilo production helpers remain usable with strict options and missing arguments.""" + (tmp_path / "relay-map.json").write_text('{"25114":"25115"}\n') + (tmp_path / "relay-map.env").write_text("PHLEX_HOST_RELAY_25114=25115\n") + (tmp_path / "kilo.json").write_text('{"url":"http://localhost:25114/v1"}\n') + command = f""" +set {options} +source "$1" +if kilo_env_run_self_tests; then exit 99; else [[ "$?" = 1 ]]; fi +kilo_env_apply_root "$2" +[[ "$PHLEX_HOST_RELAY_25114" = 25115 && "$KILO_CONFIG_CONTENT" = *host.docker.internal:25115* ]] +kilo_env_find_config_path "$2" +kilo_env_export_path_state "$2" +kilo_env_collect_exports "$2/relay-map.json" "$2/relay-map.env" "$2/kilo.json" +kilo_env_apply_paths "$2/relay-map.json" "$2/relay-map.env" "$2/kilo.json" +kilo_env_sha256 "$2/relay-map.json" +kilo_env_clear_dynamic_exports +kilo_env_clear_failed_exports +if kilo_env_apply_root "$2/incomplete"; then exit 99; else [[ "$?" = 1 ]]; fi +kilo_env_apply_root "$2/absent" +if kilo_env_sha256 "$2/absent"; then exit 99; else [[ "$?" = 1 ]]; fi +kilo_env_seed_stale_exports +kilo_env_assert_export_value PHLEX_HOST_RELAY_25114 stale fixture +kilo_env_assert_file_contains "$2/relay-map.json" 25114 fixture +kilo_env_assert_equal same same fixture +kilo_env_assert_file_empty "$2/absent" fixture +kilo_env_clear_failed_exports +kilo_env_assert_export_absent KILO_CONFIG_CONTENT fixture +hash=$(kilo_env_sha256 "$2/relay-map.json") +kilo_env_assert_hash_unchanged "$2/relay-map.json" "$hash" fixture +[[ "$(kilo_env_extract_output_value 'key=value' key)" = value ]] +if kilo_env_parse_args --invalid; then exit 99; else [[ "$?" = 1 ]]; fi +if kilo_env_run_self_tests; then exit 99; else [[ "$?" = 1 ]]; fi +for function in kilo_env_unset_var_quietly kilo_env_find_config_path kilo_env_export_path_state \ + kilo_env_collect_exports kilo_env_apply_paths kilo_env_apply_root kilo_env_sha256 \ + kilo_env_assert_equal kilo_env_assert_file_contains kilo_env_assert_file_empty \ + kilo_env_assert_export_value kilo_env_assert_export_absent kilo_env_assert_hash_unchanged \ + kilo_env_extract_output_value kilo_env_cleanup_self_test_root kilo_env_run_self_tests_impl; do + if "$function"; then exit 99; else [[ "$?" = 2 ]]; fi +done +printf 'survived\\n' +""" + incomplete = tmp_path / "incomplete" + incomplete.mkdir() + (incomplete / "relay-map.json").write_text("{}\n") + env = {key: value for key, value in os.environ.items() if key != "PHLEX_RELAY_FIXTURE_DIR"} + result = subprocess.run( + ["bash", "-c", command, "bash", str(DEVCONTAINER / "kilo-env.sh"), str(tmp_path)], + env=env, + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr + assert "unbound variable" not in result.stderr + assert "survived" in result.stdout + + +@pytest.mark.parametrize("sourced", [True, False]) +@pytest.mark.parametrize("force_sha256sum_fallback", [False, True]) +def test_kilo_env_full_self_test_under_strict_options(tmp_path, sourced, force_sha256sum_fallback): + """The complete Kilo fixture suite and its cleanup work with errexit/nounset/pipefail.""" + import hashlib + + fixture = tmp_path / "fixture" + fixture.mkdir() + (fixture / "relay-map.json").write_text('{"25114":"25115","25300":"25301"}\n') + (fixture / "relay-map.env").write_text( + "PHLEX_HOST_RELAY_25114=25115\nPHLEX_HOST_RELAY_25300=25301\n" + ) + (fixture / "artifact-sha256.txt").write_text( + "".join( + f"{hashlib.sha256((fixture / name).read_bytes()).hexdigest()} {name}\n" + for name in ("relay-map.json", "relay-map.env") + ) + ) + scratch = tmp_path / "scratch" + scratch.mkdir() + env = {**os.environ, "PHLEX_RELAY_FIXTURE_DIR": str(fixture), "TMPDIR": str(scratch)} + if force_sha256sum_fallback: + if shutil.which("sha256sum") is None: + pytest.skip("sha256sum is unavailable for fallback testing") + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + shasum = bin_dir / "shasum" + shasum.write_text("#!/bin/sh\nexit 1\n") + shasum.chmod(0o700) + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + command = ( + [ + "bash", + "-euf", + "-o", + "pipefail", + "-c", + 'source "$1" --self-test', + "bash", + str(DEVCONTAINER / "kilo-env.sh"), + ] + if sourced + else ["bash", "-euf", "-o", "pipefail", str(DEVCONTAINER / "kilo-env.sh"), "--self-test"] + ) + result = subprocess.run(command, env=env, text=True, capture_output=True, timeout=60) + assert result.returncode == 0, result.stderr + result.stdout + assert "=== Self-test passed ===" in result.stdout + assert not list(scratch.iterdir()) + + +@pytest.mark.parametrize("options", ["-e", "-u", "-eu"]) +def test_spack_renderer_failure_returns_under_strict_options(tmp_path, options): + """The sourced build helper returns for missing arguments/settings rather than exiting.""" + dockerfile = (DEVCONTAINER.parent / "ci/Dockerfile").read_text() + renderer = dockerfile.split("<<'RENDER_SPACK_TEMPLATE'\n", 1)[1].split( + "\nRENDER_SPACK_TEMPLATE", 1 + )[0] + command = ( + f"set {options}\n" + + renderer + + """ +if render_spack_template; then exit 99; else [[ "$?" = 2 ]]; fi +if render_spack_template "$1" "$2"; then exit 99; else [[ "$?" = 1 ]]; fi +printf 'survived\\n' +""" + ) + source = tmp_path / "template" + source.write_text("fixture\n") + env = {key: value for key, value in os.environ.items() if not key.startswith("PHLEX_")} + result = subprocess.run( + ["bash", "-c", command, "bash", str(source), str(tmp_path / "output")], + env=env, + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr + assert "survived" in result.stdout + assert not (tmp_path / "output").exists() + + +def test_production_kilo_sourcing_does_not_load_fixture_suite(tmp_path): + """Normal shell startup defines runtime helpers but does not parse or load the test suite.""" + command_script = "; ".join( + ( + 'source "$1"', + "declare -F kilo_env_apply_root >/dev/null", + "if declare -F kilo_env_assert_equal >/dev/null; then exit 99; fi", + "if declare -F kilo_env_run_self_tests_loaded >/dev/null; then exit 99; fi", + ) + ) + result = subprocess.run( + [ + "bash", + "-euf", + "-o", + "pipefail", + "-c", + command_script, + "bash", + str(DEVCONTAINER / "kilo-env.sh"), + ], + cwd=tmp_path, + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("helper", ["kilo-env-self-test.sh", "ensure-repos-self-test.sh"]) +def test_fixture_helpers_reject_execution(helper): + """Fixture helper libraries are source-only and do not run tests during import.""" + result = subprocess.run( + ["bash", "-eu", str(DEVCONTAINER / helper)], text=True, capture_output=True + ) + assert result.returncode != 0 + assert "sourced" in result.stderr + + +def test_kilo_test_helper_packaged_outside_profile_directory(): + """Login shells load only production code; explicit self-tests can find their helper.""" + dockerfile = (DEVCONTAINER / "Dockerfile").read_text() + assert ( + "COPY .devcontainer/kilo-env-self-test.sh /usr/local/lib/phlex/kilo-env-self-test.sh" + in dockerfile + ) + assert ( + "!.devcontainer/kilo-env-self-test.sh" + in (DEVCONTAINER.parent / ".dockerignore").read_text() + ) + + +def test_relay_production_definitions_do_not_load_fixture_suite(tmp_path): + """Relay startup and cleanup definitions remain independent of optional fixture code.""" + definitions = ( + (DEVCONTAINER / "ensure-repos.sh").read_text().split("# --- Main execution ---", 1)[0] + ) + result = subprocess.run( + ["bash"], + input=definitions + + """ +declare -F mode_cleanup_owned >/dev/null || exit 99 +if declare -F start_python_loopback_listener >/dev/null; then exit 99; fi +if declare -F mode_self_test_loaded >/dev/null; then exit 99; fi +""", + env={**os.environ, "PHLEX_RELAY_STATE_DIR": str(tmp_path)}, + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr diff --git a/scripts/test/test_promote_phlex_dependencies.py b/scripts/test/test_promote_phlex_dependencies.py new file mode 100644 index 000000000..f30a2f7a5 --- /dev/null +++ b/scripts/test/test_promote_phlex_dependencies.py @@ -0,0 +1,184 @@ +"""Test dependency-root promotion and rejection of incompatible Spack core APIs.""" + +from __future__ import annotations + +import runpy +import sys +from contextlib import nullcontext +from pathlib import Path +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest + +PROMOTE_SCRIPT = Path(__file__).resolve().parents[2] / "ci/promote_phlex_dependencies.py" +PROMOTE_MODULE = runpy.run_path(str(PROMOTE_SCRIPT)) +PROMOTE = PROMOTE_MODULE["promote_dependencies"] + + +@pytest.fixture +def environment(monkeypatch): + """Provide locked dependencies without requiring a local Spack installation.""" + spack = ModuleType("spack") + spec_module = ModuleType("spack.spec") + spec_module.__dict__["Spec"] = str + monkeypatch.setitem(sys.modules, "spack", spack) + monkeypatch.setitem(sys.modules, "spack.spec", spec_module) + + tbb = SimpleNamespace(name="intel-oneapi-tbb", dag_hash=lambda: "tbb-hash") + existing = SimpleNamespace(name="cmake", dag_hash=lambda: "cmake-hash") + phlex = SimpleNamespace( + name="phlex", + dag_hash=lambda: "phlex-hash", + dependencies=Mock(return_value=[tbb, existing]), + ) + roots = [phlex, existing] + specs = [phlex, existing, tbb] + users = set() + + def add(user_spec): + if user_spec in users: + return False + users.add(user_spec) + return True + + def add_concrete_spec(spec, concrete, *, new=True, group=None): + roots.append(concrete) + + return SimpleNamespace( + concrete_roots=lambda: iter(roots), + all_specs=lambda: iter(specs), + write_transaction=Mock(side_effect=nullcontext), + add=Mock(side_effect=add), + add_concrete_spec=Mock(wraps=add_concrete_spec), + write=Mock(), + roots=roots, + specs=specs, + users=users, + phlex=phlex, + tbb=tbb, + ) + + +def install_environment_api(monkeypatch, *environments): + """Provide the Environment constructor imported by the command entry point.""" + environment_module = ModuleType("spack.environment") + constructor = Mock(side_effect=environments) + environment_module.__dict__["Environment"] = constructor + sys.modules["spack"].__dict__["environment"] = environment_module + monkeypatch.setitem(sys.modules, "spack.environment", environment_module) + return constructor + + +def test_promotes_exact_hash_and_skips_existing_roots(environment): + """Promote only missing roots and preserve every concrete spec.""" + assert PROMOTE(environment) == ["intel-oneapi-tbb"] + environment.phlex.dependencies.assert_called_once_with(deptype=("link", "run")) + environment.add.assert_called_once_with("intel-oneapi-tbb/tbb-hash") + environment.add_concrete_spec.assert_called_once_with( + spec="intel-oneapi-tbb/tbb-hash", concrete=environment.tbb, new=True + ) + environment.write.assert_called_once_with(regenerate=False) + assert len(environment.roots) == 3 + assert len(environment.specs) == 3 + assert PROMOTE(environment) == [] + assert environment.add.call_count == 1 + + +@pytest.mark.parametrize( + "method", + [None, lambda spec: None, lambda spec, concrete, *, new, required: None], +) +def test_rejects_incompatible_api_before_mutation(environment, method): + """Missing methods and changed required arguments produce a clear API error.""" + environment.add_concrete_spec = method + with pytest.raises(RuntimeError, match=r"Unsupported Spack Environment.add_concrete_spec"): + PROMOTE(environment) + environment.write_transaction.assert_not_called() + environment.add.assert_not_called() + environment.write.assert_not_called() + + +def test_rejects_changed_api_behavior(environment): + """A signature-compatible method must actually register the concrete root.""" + environment.add_concrete_spec = lambda spec, concrete, *, new=True: None + with pytest.raises(RuntimeError, match="did not register the expected roots"): + PROMOTE(environment) + environment.write.assert_not_called() + + +def test_rejects_changed_concrete_dag(environment): + """Promotion must not remove or replace any locked dependency.""" + + def add_concrete_spec(spec, concrete, *, new=True): + environment.roots.append(concrete) + environment.specs.remove(concrete) + + environment.add_concrete_spec = add_concrete_spec + with pytest.raises(RuntimeError, match="unexpectedly changed the concrete DAG"): + PROMOTE(environment) + environment.write.assert_not_called() + + +@pytest.mark.parametrize("root_count", [0, 2]) +def test_rejects_environment_without_exactly_one_phlex_root(environment, root_count): + """Reject missing or duplicate Phlex roots before changing the environment.""" + environment.roots[:] = [environment.phlex] * root_count + + with pytest.raises(RuntimeError, match="exactly one concrete Phlex root"): + PROMOTE(environment) + + environment.write_transaction.assert_not_called() + environment.add.assert_not_called() + + +def test_rejects_dependency_that_spack_cannot_add_as_a_root(environment): + """Fail before concrete registration if Spack rejects the exact-hash root.""" + environment.add.side_effect = lambda _: False + + with pytest.raises(RuntimeError, match="already an unconcretized root"): + PROMOTE(environment) + + environment.add.assert_called_once_with("intel-oneapi-tbb/tbb-hash") + environment.add_concrete_spec.assert_not_called() + environment.write.assert_not_called() + + +def test_main_promotes_and_verifies_persisted_environment(environment, monkeypatch, capsys): + """Run the command entry point and confirm persisted roots and DAG hashes.""" + env_path = "/tmp/phlex-spack-env" + monkeypatch.setenv("PHLEX_SPACK_ENV", env_path) + persisted = SimpleNamespace( + concrete_roots=lambda: iter(environment.roots), + all_specs=lambda: iter(environment.specs), + ) + constructor = install_environment_api(monkeypatch, environment, persisted) + + runpy.run_path(str(PROMOTE_SCRIPT), run_name="__main__") + + assert [call.args for call in constructor.call_args_list] == [(env_path,), (env_path,)] + assert "Promoted Phlex dependencies: intel-oneapi-tbb" in capsys.readouterr().out + + +@pytest.mark.parametrize( + ("persisted_roots", "persisted_specs", "error_message"), + [ + (False, True, "did not persist the promoted dependency roots"), + (True, False, "changed the concrete DAG while writing the environment"), + ], +) +def test_main_rejects_inconsistent_persisted_environment( + environment, monkeypatch, persisted_roots, persisted_specs, error_message +): + """Reject a persisted environment that loses roots or changes its DAG.""" + monkeypatch.setenv("PHLEX_SPACK_ENV", "/tmp/phlex-spack-env") + roots = environment.roots if persisted_roots else environment.roots[:-1] + specs = environment.specs if persisted_specs else environment.specs[:-1] + persisted = SimpleNamespace( + concrete_roots=lambda: iter(roots), + all_specs=lambda: iter(specs), + ) + install_environment_api(monkeypatch, environment, persisted) + + with pytest.raises(SystemExit, match=error_message): + PROMOTE_MODULE["main"]() diff --git a/scripts/test/test_upgrade_repos.py b/scripts/test/test_upgrade_repos.py new file mode 100644 index 000000000..c8dc5f5a7 --- /dev/null +++ b/scripts/test/test_upgrade_repos.py @@ -0,0 +1,223 @@ +"""Regression tests for upgrading Spack repositories and package imports.""" + +from __future__ import annotations + +import runpy +import sys +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest +import yaml + +UPGRADE_SCRIPT = Path(__file__).resolve().parents[2] / "ci/upgrade_repos.py" + + +def load_upgrade_script(monkeypatch: pytest.MonkeyPatch): + """Load the CI script with a PyYAML-backed stand-in for Spack's ruamel API.""" + spack = ModuleType("spack") + spack.__path__ = [] + repo_module = ModuleType("spack.repo") + repo_module.__dict__["PATH"] = SimpleNamespace(repos=[]) + repo_module.__dict__["Repo"] = object + spack.__dict__["repo"] = repo_module + + vendor = ModuleType("spack.vendor") + vendor.__path__ = [] + ruamel = ModuleType("spack.vendor.ruamel") + ruamel.__path__ = [] + yaml_module = ModuleType("spack.vendor.ruamel.yaml") + + class YAML: + preserve_quotes = False + + @staticmethod + def load(stream): + with stream.open(encoding="utf-8") as source: + return yaml.safe_load(source) + + @staticmethod + def dump(data, stream): + yaml.safe_dump(data, stream, sort_keys=False) + + yaml_module.__dict__["YAML"] = YAML + monkeypatch.setitem(sys.modules, "spack", spack) + monkeypatch.setitem(sys.modules, "spack.repo", repo_module) + monkeypatch.setitem(sys.modules, "spack.vendor", vendor) + monkeypatch.setitem(sys.modules, "spack.vendor.ruamel", ruamel) + monkeypatch.setitem(sys.modules, "spack.vendor.ruamel.yaml", yaml_module) + return runpy.run_path(str(UPGRADE_SCRIPT)) + + +def make_repo(root: Path, namespace: str = "test") -> SimpleNamespace: + """Create the minimal repository object expected by the migration helpers.""" + return SimpleNamespace(root=str(root), namespace=namespace) + + +def test_upgrade_repo_api_migrates_old_version_and_preserves_other_fields( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Upgrade legacy metadata without discarding unrelated repository fields.""" + upgrade = load_upgrade_script(monkeypatch) + repo = make_repo(tmp_path, "custom") + repo_yaml = tmp_path / "repo.yaml" + repo_yaml.write_text( + 'repo:\n namespace: custom\n api: "v2.0"\nmetadata:\n owner: local\n', + encoding="utf-8", + ) + + upgrade["upgrade_spack_repo_api"](repo) + + result = yaml.safe_load(repo_yaml.read_text(encoding="utf-8")) + assert result == { + "repo": {"namespace": "custom", "api": "v2.2"}, + "metadata": {"owner": "local"}, + } + assert "Upgrading custom from API v2.0 to v2.2" in capsys.readouterr().out + + +@pytest.mark.parametrize("api", ["v2.2", "v3.0"]) +def test_upgrade_repo_api_leaves_current_and_newer_files_unchanged( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, api: str +) -> None: + """Avoid rewriting repositories already on the target or a newer API.""" + upgrade = load_upgrade_script(monkeypatch) + repo_yaml = tmp_path / "repo.yaml" + original = f"repo:\n api: {api}\n owner: preserved\n" + repo_yaml.write_text(original, encoding="utf-8") + + upgrade["upgrade_spack_repo_api"](make_repo(tmp_path)) + + assert repo_yaml.read_text(encoding="utf-8") == original + + +def test_upgrade_repo_api_ignores_missing_file_and_missing_repo_section( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Ignore repositories without a repo.yaml or a repo metadata section.""" + upgrade = load_upgrade_script(monkeypatch) + upgrade["upgrade_spack_repo_api"](make_repo(tmp_path)) + + repo_yaml = tmp_path / "repo.yaml" + repo_yaml.write_text("metadata:\n owner: local\n", encoding="utf-8") + upgrade["upgrade_spack_repo_api"](make_repo(tmp_path)) + assert yaml.safe_load(repo_yaml.read_text(encoding="utf-8")) == { + "metadata": {"owner": "local"} + } + + +def test_upgrade_repo_api_reports_invalid_yaml_with_path_context( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Wrap parser failures with the path of the repository metadata file.""" + upgrade = load_upgrade_script(monkeypatch) + repo_yaml = tmp_path / "repo.yaml" + repo_yaml.write_text("repo: [unterminated", encoding="utf-8") + + with pytest.raises(RuntimeError, match="Error updating") as error: + upgrade["upgrade_spack_repo_api"](make_repo(tmp_path)) + + assert error.value.__cause__ is not None + assert str(repo_yaml) in str(error.value) + + +def test_clean_package_imports_rewrites_nested_recipes_only( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Rewrite legacy imports in nested recipes, not adjacent Python or docs.""" + upgrade = load_upgrade_script(monkeypatch) + packages = tmp_path / "packages" + recipe = packages / "group" / "package.py" + recipe.parent.mkdir(parents=True) + recipe.write_text( + "import llnl.util.filesystem as filesystem\n" + "from llnl.util.filesystem import working_dir\n" + "from spack.llnl.util import filesystem\n" + "from spack.llnl.util.filesystem import mkdirp\n" + "import spack.llnl.util.filesystem as legacy\n", + encoding="utf-8", + ) + unrelated_python = packages / "group" / "helper.py" + unrelated_python.write_text("import llnl.util.filesystem as filesystem\n", encoding="utf-8") + documentation = packages / "group" / "README.md" + documentation.write_text("import llnl.util.filesystem as filesystem\n", encoding="utf-8") + + upgrade["clean_package_imports"](make_repo(tmp_path, "custom")) + + assert recipe.read_text(encoding="utf-8") == ( + "import spack.util.filesystem as filesystem\n" + "from spack.util.filesystem import working_dir\n" + "import spack.util.filesystem as filesystem\n" + "from spack.util.filesystem import mkdirp\n" + "import spack.util.filesystem as legacy\n" + ) + assert ( + unrelated_python.read_text(encoding="utf-8") + == "import llnl.util.filesystem as filesystem\n" + ) + assert ( + documentation.read_text(encoding="utf-8") == "import llnl.util.filesystem as filesystem\n" + ) + output = capsys.readouterr().out + assert "group/package.py" in output + assert "Successfully patched 1 package recipes." in output + + +def test_clean_package_imports_does_not_rewrite_already_clean_recipe( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Leave an already-canonical recipe byte-for-byte unchanged.""" + upgrade = load_upgrade_script(monkeypatch) + packages = tmp_path / "packages" + packages.mkdir() + recipe = packages / "package.py" + original = "from spack.util.filesystem import mkdirp\n" + recipe.write_text(original, encoding="utf-8") + + upgrade["clean_package_imports"](make_repo(tmp_path)) + + assert recipe.read_text(encoding="utf-8") == original + + +def test_clean_package_imports_ignores_missing_package_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Skip scanning when a repository has no package directory.""" + upgrade = load_upgrade_script(monkeypatch) + upgrade["clean_package_imports"](make_repo(tmp_path)) + assert capsys.readouterr().out == "" + + +def test_main_skips_builtin_repo_and_upgrades_custom_repo( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Protect builtin Spack while migrating active third-party repositories.""" + upgrade = load_upgrade_script(monkeypatch) + builtin = tmp_path / "builtin" + custom = tmp_path / "custom" + builtin.mkdir() + custom.mkdir() + builtin_yaml = builtin / "repo.yaml" + builtin_yaml.write_text("repo:\n api: v1.0\n", encoding="utf-8") + builtin_recipe = builtin / "packages" / "builtin" / "package.py" + builtin_recipe.parent.mkdir(parents=True) + builtin_recipe.write_text("import llnl.util.filesystem as filesystem\n", encoding="utf-8") + custom_yaml = custom / "repo.yaml" + custom_yaml.write_text("repo:\n api: v1.0\n", encoding="utf-8") + custom_recipe = custom / "packages" / "custom" / "package.py" + custom_recipe.parent.mkdir(parents=True) + custom_recipe.write_text("import llnl.util.filesystem as filesystem\n", encoding="utf-8") + + upgrade["spack"].repo.PATH.repos = [make_repo(builtin, "builtin"), make_repo(custom, "custom")] + upgrade["main"]() + + assert yaml.safe_load(builtin_yaml.read_text(encoding="utf-8"))["repo"]["api"] == "v1.0" + assert yaml.safe_load(custom_yaml.read_text(encoding="utf-8"))["repo"]["api"] == "v2.2" + assert ( + builtin_recipe.read_text(encoding="utf-8") == "import llnl.util.filesystem as filesystem\n" + ) + assert ( + custom_recipe.read_text(encoding="utf-8") == "import spack.util.filesystem as filesystem\n" + ) diff --git a/scripts/test/test_workflow_cleanup.py b/scripts/test/test_workflow_cleanup.py index 25c59673a..b4c92a532 100644 --- a/scripts/test/test_workflow_cleanup.py +++ b/scripts/test/test_workflow_cleanup.py @@ -6,8 +6,11 @@ YAML structure and markers are present. """ +import os import pathlib +import subprocess +import pytest import yaml WORKFLOWS_DIR = pathlib.Path(__file__).resolve().parent.parent.parent / ".github" / "workflows" @@ -77,3 +80,48 @@ def test_clang_tidy_marks_completed_only_after_tidy_succeeds(): ) assert tidy_failure < issues_found < completed_marker + + +@pytest.mark.parametrize( + "coverage_result,available,expected_status", + [ + ("success", ("amd64", "arm64"), 0), + ("success", ("amd64",), 1), + ("success", ("arm64",), 1), + ("success", (), 1), + ("failure", ("amd64",), 0), + ("failure", ("arm64",), 0), + ("failure", (), 1), + ], +) +def test_coverage_upload_requires_reports_from_successful_matrix( + tmp_path, coverage_result, available, expected_status +): + """Missing successful reports fail collection without hiding available upload targets.""" + workflow = _load_yaml(WORKFLOWS_DIR / "coverage.yaml") + steps = workflow["jobs"]["coverage-upload"]["steps"] + downloads = [step for step in steps if step["name"].startswith("Download ")] + assert len(downloads) == 2 + assert all(step["continue-on-error"] and step["if"] == "always()" for step in downloads) + collector = next(step for step in steps if step.get("id") == "codecov_targets") + assert collector["env"]["COVERAGE_RESULT"] == "${{ needs.coverage.result }}" + for arch in available: + report_dir = tmp_path / "coverage-artifacts" / f"coverage-generated-{arch}" + report_dir.mkdir(parents=True) + (report_dir / "coverage.xml").write_text("\n") + output = tmp_path / "output" + result = subprocess.run( + ["bash", "-c", collector["run"]], + cwd=tmp_path, + env={**os.environ, "GITHUB_OUTPUT": str(output), "COVERAGE_RESULT": coverage_result}, + text=True, + capture_output=True, + ) + assert result.returncode == expected_status, result.stderr + targets = dict(line.split("=", 1) for line in output.read_text().splitlines()) + for arch in ("amd64", "arm64"): + assert targets[f"{arch}_found"] == str(arch in available).lower() + if arch in available: + assert targets[f"{arch}_files"].endswith("/coverage.xml") + uploads = [step for step in steps if step["name"].endswith("coverage to Codecov")] + assert all("always()" in step["if"] for step in uploads)