From f51e4262f072b53322aeb914b9aea98653a65308 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:09:43 -0500 Subject: [PATCH 1/5] feat(native): build Compose build services in native up behind HACK_NATIVE_BUILDS Native foreground up can now build Compose `build:` services in the native runtime when HACK_NATIVE_BUILDS=1 is set. Without it, a build service keeps refusing before any hook or VM effect, now with a fixed message that names the variable. - Pure admission before hooks or VM start: `build: ` or a mapping of context, dockerfile, target and literal args (mapping of strings or numbers, or KEY=VALUE list). Compose path rules (context relative to the Compose file, Dockerfile relative to the context), inside the project. Interpolation, host-variable inheritance, null values, other build keys, remote contexts and image+build refuse; values are never shown. - After runtime up, each distinct build runs once through runtime build-image with its output streamed to stderr; every service with that build is pinned to the returned content ID before image pinning and review. The request file is private (0600, removed after use). Cancellation sends SIGTERM so the runtime stops its build client group, then SIGKILL after a grace period. - HACK_NATIVE_REBUILD=1 asks for a rebuild. Restore of an unchanged retained run uses retained images and never builds (a missing one refuses); hack restart never builds and refuses before stopping anything when a build service has no running or retained image. Refs HACK-1270 Co-Authored-By: Claude Opus 5.5 --- docs/guides/native-candidate.md | 39 +- packages/runtime-core/README.md | 4 +- src/backends/native-project-builds.ts | 561 ++++++++++++++++++ .../native-project-image-admission.ts | 31 +- .../native-project-restart-preflight.ts | 27 +- src/backends/native-project-start.ts | 90 ++- ...ative-compose-backend-features-cli.test.ts | 18 +- tests/native-project-builds.test.ts | 331 +++++++++++ 8 files changed, 1066 insertions(+), 35 deletions(-) create mode 100644 src/backends/native-project-builds.ts create mode 100644 tests/native-project-builds.test.ts diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 9f4f986fb..41191b5be 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -228,9 +228,33 @@ selection; see [Test both backends](https://github.com/hack-dance/hack/blob/next Other labels keep their native behavior. Routing (`caddy*`) and dependency cache (`hack.dependencies.*`) labels are reviewed as before, and other ordinary labels -are refused with the general native configuration refusal. A basic build is also -refused by native image-project `up`, with the general refusal: it needs the -authored native format. +are refused with the general native configuration refusal. + +Native image-project `up` builds Compose `build:` services only when you opt in +with `HACK_NATIVE_BUILDS=1` (and the usual `HACK_NATIVE_SHARED_SOURCE=1`). Without +it, a `build:` service refuses before any hook or VM effect and names the variable. +With it: + +- Accepted forms are `build: ` and a mapping of `context`, `dockerfile`, + `target` and `args`. Paths follow Compose (the context is relative to the Compose + file, the Dockerfile to the context) and must stay inside the project. `args` may be + a mapping of literal strings or numbers, or a list of `KEY=VALUE` literals. Any `$` + interpolation, a list item without `=` (host variable inheritance), `null`, other + build keys (`secrets`, `ssh`, `cache_from`, `platforms`, `network` and the rest), + remote contexts and `image` together with `build` refuse. Values are never shown. +- After the runtime is up, each distinct build runs once with + `runtime build-image` (below), and its output streams to your terminal. Every + service with that build is then pinned to the built content ID, so review, readiness + and restore treat it like an image service. +- An existing image for the same build inputs is reused without building. Set + `HACK_NATIVE_REBUILD=1` to build again; base-image tags then resolve again too. + Edits to other files in the context do not trigger a rebuild, as with Compose. +- Restoring an unchanged retained run uses the retained images and never builds; if + one is missing, `up` refuses and asks you to start fresh. `hack restart` also never + builds: it reuses the running or retained image, or refuses before stopping anything. +- Services in profiles you did not select are built too, because the native review + pins every declared service. Native builds are `linux/arm64` only; + `platform: linux/amd64` is still refused. `hack doctor` reports the same features for the current project in its `compose backend features` check. With `--json`, the check's `composeBackend.features` @@ -443,9 +467,8 @@ Provider setup is separate and explicit. Obtain the exact archives identified in ./hack-native --candidate-root /absolute/private/candidate-home runtime probe --json ``` -The pins also name a BuildKit client (`buildx`, `darwin-arm64`) for future native image -builds. Native `up` still refuses `build:` services and nothing runs this client yet. -Preparing it is optional: +The pins also name a BuildKit client (`buildx`, `darwin-arm64`) for native image +builds. Prepare it before using `HACK_NATIVE_BUILDS=1`; it is optional otherwise: ```sh ./hack-native --candidate-root /absolute/private/candidate-home runtime prepare-builder --archive /absolute/docker-buildx @@ -458,8 +481,8 @@ it again and does not replace it. If the prepared tree changes, `runtime probe` refuses; it reports `builder_prepared` when the client is present and unchanged. `runtime build-image --request --json` builds one image in the running -runtime's engine with that client. It is a building block: native `up` still refuses -`build:` services and does not call it. The request is a private JSON file (a singly +runtime's engine with that client. Native `up` calls it for `build:` services when +`HACK_NATIVE_BUILDS=1` is set; you can also run it directly. The request is a private JSON file (a singly linked, mode-`0600` file owned by you), because build-argument values are in it: ```json diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index aca2f2762..3f650f695 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -489,8 +489,8 @@ that does not name an owned network or `none`. Cold builds need free guest stora (at least 6 GiB, or twice the key's last image), successful builds prune BuildKit cache to 8 GiB, and `runtime reclaim-build-images` removes only owned, superseded, unreferenced images (`provider/image_build/reclaim.rs`; model -`tests/models/tla/native-build-retention`). Native `up` does not call the build -operation yet. +`tests/models/tla/native-build-retention`). Native `up` calls the build operation +for Compose `build:` services only when `HACK_NATIVE_BUILDS=1` is set. Source binaries embed their build checkout. If a copied Cargo target directory retains a different checkout identity, use a fresh `--target-dir` for checks and diff --git a/src/backends/native-project-builds.ts b/src/backends/native-project-builds.ts new file mode 100644 index 000000000..369b65e2a --- /dev/null +++ b/src/backends/native-project-builds.ts @@ -0,0 +1,561 @@ +import { mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { ClosedCliError, HackCliError } from "../lib/cli-result.ts"; +import { isRecord } from "../lib/guards.ts"; +import { + NativeRuntimeRequestError, + type NativeRuntimeSelection, +} from "./native-runtime-client.ts"; + +/** + * Native Compose `build:` services (feature-gated by `HACK_NATIVE_BUILDS=1`). + * + * Admission is pure and runs before any hook or VM effect: it accepts the Compose + * forms the native runtime builds (`build: ` or a mapping of `context`, + * `dockerfile`, `target` and literal `args`) and refuses everything else with a + * closed, coded refusal. After the runtime is up, each distinct build runs once + * through `runtime build-image`; the service is then pinned to the returned content + * ID, so review, graph admission, receipts and restore treat it as an image + * service. Restore and restart never build: they use the retained content ID or + * refuse before any graph effect. + */ + +const IMAGE_ID = /^sha256:[a-f0-9]{64}$/; +const BUILD_KEY = /^[a-f0-9]{64}$/; +const ARGUMENT = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; +const TARGET = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; +const BUILD_KEYS = new Set(["context", "dockerfile", "target", "args"]); +const MAX_ARGS = 64; +/** The runtime's own budget is 30 minutes; the client waits a little longer. */ +const BUILD_TIMEOUT_SECONDS = 30 * 60; +/** After SIGTERM the runtime stops its build client within ten seconds. */ +const CANCEL_GRACE_MS = 20_000; +const MAX_STDOUT = 64 * 1024; +const ERROR_CODE = /^[a-z][a-z0-9_]{0,63}$/; + +/** Whether native `up` may build Compose `build:` services. */ +export function nativeBuildsEnabled( + env: Readonly> = process.env +): boolean { + return env.HACK_NATIVE_BUILDS === "1"; +} + +/** Whether `HACK_NATIVE_REBUILD=1` asks every build to run again. */ +export function nativeRebuildRequested( + env: Readonly> = process.env +): boolean { + return env.HACK_NATIVE_REBUILD === "1"; +} + +/** One admitted build: paths relative to the project root and the context. */ +export interface NativeBuildRequest { + readonly context: string; + readonly dockerfile: string; + readonly target: string | null; + /** Literal values. Never log, print or persist outside the private request file. */ + readonly args: Readonly>; +} + +/** Fixed refusal for a `build:` service while native builds are not enabled. */ +export const NATIVE_BUILDS_DISABLED_REFUSAL = + "Native foreground up builds Compose build services only with HACK_NATIVE_BUILDS=1; set it to build them in the native runtime. No hook, VM or build was started."; + +function refused(detail: string): HackCliError { + return new HackCliError({ + code: "E_NATIVE_PROJECT_UNSUPPORTED", + message: `Native foreground up cannot build this service: ${detail} Values were omitted; no hook, VM or build was started.`, + }); +} + +function literal(value: unknown, what: string): string { + if (typeof value !== "string" || value.length === 0 || value.includes("$")) { + throw refused( + `${what} must be a literal non-empty string without Compose interpolation.` + ); + } + return value; +} + +function argumentEntries(value: unknown): [string, unknown][] { + if (isRecord(value)) { + return Object.entries(value); + } + if (!Array.isArray(value)) { + throw refused("build args must be a mapping or a list."); + } + return value.map((item: unknown): [string, unknown] => { + if (typeof item !== "string" || !item.includes("=")) { + throw refused( + "list-form build args must be KEY=VALUE literals; inheriting a host variable is not supported." + ); + } + const equal = item.indexOf("="); + return [item.slice(0, equal), item.slice(equal + 1)]; + }); +} + +function buildArguments(value: unknown): Record { + if (value === undefined) { + return {}; + } + const entries = argumentEntries(value); + if (entries.length > MAX_ARGS) { + throw refused("a native build accepts at most 64 build args."); + } + const result: Record = {}; + for (const [name, raw] of entries) { + if (!ARGUMENT.test(name) || Object.hasOwn(result, name)) { + throw refused("build arg names must be unique identifiers."); + } + const text = + typeof raw === "number" && Number.isFinite(raw) ? String(raw) : raw; + if (typeof text !== "string" || text.includes("$")) { + // Env-derived values would be baked into image history. + throw refused( + "build arg values must be literal strings or numbers; interpolation and null values are not supported." + ); + } + result[name] = text; + } + return result; +} + +/** The declared build fields with Compose defaults, before path checks. */ +function buildFields(build: unknown): NativeBuildRequest { + if (typeof build === "string") { + return { + context: literal(build, "the build context"), + dockerfile: "Dockerfile", + target: null, + args: {}, + }; + } + if (!isRecord(build)) { + throw refused("build must be a context path or a mapping."); + } + if (Object.keys(build).some((key) => !BUILD_KEYS.has(key))) { + throw refused( + "only build context, dockerfile, target and literal args are supported." + ); + } + const target = + build.target === undefined + ? null + : literal(build.target, "the build target"); + if (target !== null && !TARGET.test(target)) { + throw refused("the build target must be a stage name."); + } + return { + context: + build.context === undefined + ? "." + : literal(build.context, "the build context"), + dockerfile: + build.dockerfile === undefined + ? "Dockerfile" + : literal(build.dockerfile, "the Dockerfile path"), + target, + args: buildArguments(build.args), + }; +} + +/** + * Admit one service's `build` declaration, or return `undefined` when it has none. + * Paths follow Compose: the context is relative to the Compose file's directory and + * the Dockerfile is relative to the context. The result is project-relative; the + * runtime resolves it again without following symlinks. + */ +export function admitNativeProjectBuild(opts: { + readonly service: Readonly>; + readonly projectRoot: string; + readonly composeFile: string; +}): NativeBuildRequest | undefined { + const build = opts.service.build; + if (build === undefined) { + return; + } + if (Object.hasOwn(opts.service, "image")) { + throw refused("a service may not declare both image and build."); + } + const { context, dockerfile, target, args } = buildFields(build); + if (context.includes("://") || context.startsWith("git@")) { + throw refused("remote build contexts are not supported."); + } + if (isAbsolute(dockerfile)) { + throw refused("the Dockerfile must be relative to the build context."); + } + const absolute = resolve(dirname(opts.composeFile), context); + const root = resolve(opts.projectRoot); + const inside = relative(root, absolute); + if (inside === ".." || inside.startsWith(`..${sep}`) || isAbsolute(inside)) { + throw refused("the build context must be inside the project."); + } + return { context: inside === "" ? "." : inside, dockerfile, target, args }; +} + +/** + * Admit every build service of a normalized Compose document. Throws the closed + * refusal for any unsupported form, and when builds are not enabled. + */ +export function admitNativeProjectBuilds(opts: { + readonly services: Readonly< + Record>> + >; + readonly projectRoot: string; + readonly composeFile: string; + readonly enabled: boolean; +}): ReadonlyMap { + const builds = new Map(); + for (const [name, service] of Object.entries(opts.services)) { + const request = admitNativeProjectBuild({ ...opts, service }); + if (request === undefined) { + continue; + } + if (!opts.enabled) { + throw new HackCliError({ + code: "E_NATIVE_PROJECT_UNSUPPORTED", + message: NATIVE_BUILDS_DISABLED_REFUSAL, + }); + } + builds.set(name, request); + } + return builds; +} + +/** The result of one `runtime build-image`, checked against its closed shape. */ +export interface NativeBuiltImage { + readonly imageId: string; + readonly buildKey: string; + readonly reused: boolean; +} + +function parseBuilt(value: unknown): NativeBuiltImage { + if ( + !( + isRecord(value) && + typeof value.image_id === "string" && + IMAGE_ID.test(value.image_id) && + typeof value.build_key === "string" && + BUILD_KEY.test(value.build_key) && + typeof value.reused === "boolean" + ) + ) { + throw new ClosedCliError( + "Native runtime returned an invalid image build result; values omitted." + ); + } + return { + imageId: value.image_id, + buildKey: value.build_key, + reused: value.reused, + }; +} + +/** + * Forward build output to stderr as it arrives, holding back a final structured + * runtime error line so it can be returned instead of printed. + */ +async function forwardBuildOutput( + stream: ReadableStream, + write: (text: string) => void +): Promise { + const decoder = new TextDecoder(); + let pending = ""; + let held: string | undefined; + const emit = (line: string) => { + if (held !== undefined) { + write(`${held}\n`); + held = undefined; + } + if (line.startsWith('{"code":')) { + held = line; + } else { + write(`${line}\n`); + } + }; + const reader = stream.getReader(); + while (true) { + const { done, value: chunk } = await reader.read(); + if (done) { + break; + } + pending += decoder.decode(chunk, { stream: true }); + let newline = pending.indexOf("\n"); + while (newline >= 0) { + emit(pending.slice(0, newline)); + pending = pending.slice(newline + 1); + newline = pending.indexOf("\n"); + } + } + pending += decoder.decode(); + if (pending.length > 0) { + emit(pending); + } + return held; +} + +function failureCode(line: string | undefined): string | undefined { + if (line === undefined) { + return; + } + try { + const value: unknown = JSON.parse(line); + return isRecord(value) && + typeof value.code === "string" && + ERROR_CODE.test(value.code) + ? value.code + : undefined; + } catch { + return; + } +} + +function failureMessage(line: string | undefined): string | undefined { + try { + const value: unknown = line === undefined ? undefined : JSON.parse(line); + return isRecord(value) && typeof value.message === "string" + ? value.message.slice(0, 512) + : undefined; + } catch { + return; + } +} + +/** + * Run `runtime build-image` with build output streamed to stderr. Cancellation sends + * SIGTERM so the runtime can stop its build client's process group, then SIGKILL + * after a grace period. The request file path is the only argument; it carries the + * argument values and is private to this invocation. + */ +export async function invokeNativeBuild(opts: { + readonly runtime: NativeRuntimeSelection; + readonly cwd: string; + readonly requestFile: string; + readonly signal?: AbortSignal; + readonly write?: (text: string) => void; +}): Promise { + if (opts.signal?.aborted) { + throw new ClosedCliError( + "Native foreground up was canceled; values omitted." + ); + } + const child = Bun.spawn( + [ + opts.runtime.binary, + "--candidate-root", + opts.runtime.home, + "runtime", + "build-image", + "--request", + opts.requestFile, + "--json", + ], + { + cwd: opts.cwd, + env: systemEnvironment(), + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + } + ); + let killTimer: ReturnType | undefined; + let canceled = false; + const cancel = () => { + canceled = true; + child.kill("SIGTERM"); + killTimer = setTimeout(() => child.kill("SIGKILL"), CANCEL_GRACE_MS); + }; + opts.signal?.addEventListener("abort", cancel, { once: true }); + const outer = setTimeout(cancel, (BUILD_TIMEOUT_SECONDS + 60) * 1000); + try { + const [stdout, held, code] = await Promise.all([ + new Response(child.stdout).arrayBuffer(), + forwardBuildOutput( + child.stderr, + opts.write ?? ((text) => process.stderr.write(text)) + ), + child.exited, + ]); + if (canceled) { + throw new ClosedCliError( + "Native foreground up was canceled during an image build; no image was adopted." + ); + } + if (code !== 0) { + const nativeCode = failureCode(held); + throw new NativeRuntimeRequestError({ + message: + failureMessage(held) ?? + "Native image build failed; its output is shown above.", + nativeCode, + }); + } + if (stdout.byteLength > MAX_STDOUT) { + throw new ClosedCliError( + "Native runtime returned an oversized image build result; values omitted." + ); + } + return parseBuilt(JSON.parse(new TextDecoder().decode(stdout))); + } finally { + opts.signal?.removeEventListener("abort", cancel); + clearTimeout(outer); + clearTimeout(killTimer); + if (child.exitCode === null) { + child.kill("SIGKILL"); + await child.exited; + } + } +} + +/** The same minimal process environment ordinary runtime requests receive. */ +function systemEnvironment(): Record { + const env: Record = {}; + for (const key of ["PATH", "HOME", "TMPDIR", "USER", "LOGNAME"]) { + const value = process.env[key]; + if (value !== undefined) { + env[key] = value; + } + } + return env; +} + +/** A refusal when a retained run's built image is gone: restore never builds. */ +function retainedMissing(service: string): ClosedCliError { + return new ClosedCliError( + `The retained built image for ${service} was removed; start fresh instead of restoring. Nothing was built.` + ); +} + +/** + * Build (or reuse) every admitted build service and pin it to its content ID. + * Identical declarations build once. Retained content IDs always win, and a + * restore with a missing retained image refuses instead of building. + */ +export async function buildNativeProjectImages(opts: { + readonly runtime: NativeRuntimeSelection; + readonly projectRoot: string; + readonly branch: string | null | undefined; + readonly builds: ReadonlyMap; + readonly specs: Record>; + readonly retainedImages: ReadonlyMap; + readonly restore: boolean; + readonly rebuild: boolean; + readonly signal: AbortSignal; + readonly invoke?: typeof invokeNativeBuild; +}): Promise { + const pending = pendingBuilds(opts); + if (pending.size === 0) { + return; + } + await runPendingBuilds({ ...opts, pending }); +} + +type PendingBuild = { request: NativeBuildRequest; services: string[] }; + +/** Pin retained images, refuse a restore without one, and group identical builds. */ +function pendingBuilds(opts: { + readonly builds: ReadonlyMap; + readonly specs: Record>; + readonly retainedImages: ReadonlyMap; + readonly restore: boolean; +}): Map { + const pending = new Map(); + for (const [service, request] of opts.builds) { + const spec = opts.specs[service]; + if (!spec) { + throw new ClosedCliError( + "Native build services changed during startup; values omitted." + ); + } + const retained = opts.retainedImages.get(service); + if (retained !== undefined) { + pin(spec, retained); + continue; + } + // A retained run selects retained images only while its configuration is + // unchanged; then a build service without one refuses instead of building. + if (opts.restore && opts.retainedImages.size > 0) { + throw retainedMissing(service); + } + const identity = JSON.stringify([ + request.context, + request.dockerfile, + request.target, + Object.entries(request.args).sort(([a], [b]) => (a < b ? -1 : 1)), + ]); + const entry = pending.get(identity) ?? { request, services: [] }; + entry.services.push(service); + pending.set(identity, entry); + } + return pending; +} + +async function runPendingBuilds(opts: { + readonly runtime: NativeRuntimeSelection; + readonly projectRoot: string; + readonly branch: string | null | undefined; + readonly pending: ReadonlyMap; + readonly specs: Record>; + readonly rebuild: boolean; + readonly signal: AbortSignal; + readonly invoke?: typeof invokeNativeBuild; +}): Promise { + const invoke = opts.invoke ?? invokeNativeBuild; + // The runtime requires the exact canonical root; it also derives the namespace. + const projectRoot = await realpath(opts.projectRoot); + const directory = await mkdtemp(join(tmpdir(), "hack-native-build-")); + try { + let index = 0; + for (const { request, services } of opts.pending.values()) { + if (opts.signal.aborted) { + throw new ClosedCliError( + "Native foreground up was canceled; values omitted." + ); + } + const requestFile = join(directory, `request-${index}.json`); + index += 1; + await writeFile( + requestFile, + JSON.stringify({ + version: 1, + project_root: projectRoot, + branch: opts.branch ?? null, + context: request.context, + dockerfile: request.dockerfile, + target: request.target, + args: request.args, + policy: opts.rebuild ? "rebuild" : "missing", + timeout_seconds: BUILD_TIMEOUT_SECONDS, + }), + { mode: 0o600, flag: "wx" } + ); + const built = await invoke({ + runtime: opts.runtime, + cwd: opts.projectRoot, + requestFile, + signal: opts.signal, + }); + // The request carries argument values; remove it as soon as it is consumed. + await rm(requestFile, { force: true }); + for (const service of services) { + const spec = opts.specs[service]; + if (spec) { + pin(spec, built.imageId); + } + } + } + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +function pin(spec: Record, image: string): void { + if (!IMAGE_ID.test(image)) { + throw new ClosedCliError( + "Native build image selection is invalid; values omitted." + ); + } + spec.image = image; + Reflect.deleteProperty(spec, "build"); +} diff --git a/src/backends/native-project-image-admission.ts b/src/backends/native-project-image-admission.ts index 3e032ef2f..b589ea3df 100644 --- a/src/backends/native-project-image-admission.ts +++ b/src/backends/native-project-image-admission.ts @@ -34,6 +34,27 @@ function canceled(): ClosedCliError { ); } +/** Distinct registry references to admit; content IDs and admitted builds need none. */ +function imageReferences( + specs: Readonly>>> +): Set { + const references = new Set(); + for (const spec of Object.values(specs)) { + // An admitted build service has no image yet; its base images are pinned + // by the runtime build itself. + if (spec.image === undefined && spec.build !== undefined) { + continue; + } + if (typeof spec.image !== "string") { + throw refused(); + } + if (!IMAGE_ID.test(spec.image)) { + references.add(spec.image); + } + } + return references; +} + /** * Admit every authored service image before any VM, provider or image effect. * @@ -53,15 +74,7 @@ export async function admitNativeProjectImages(opts: { readonly invoke?: typeof invokeNativeRuntime; }): Promise { const invoke = opts.invoke ?? invokeNativeRuntime; - const references = new Set(); - for (const spec of Object.values(opts.specs)) { - if (typeof spec.image !== "string") { - throw refused(); - } - if (!IMAGE_ID.test(spec.image)) { - references.add(spec.image); - } - } + const references = imageReferences(opts.specs); for (const reference of references) { if (opts.signal?.aborted) { throw canceled(); diff --git a/src/backends/native-project-restart-preflight.ts b/src/backends/native-project-restart-preflight.ts index cd7ac3ce4..550881f8d 100644 --- a/src/backends/native-project-restart-preflight.ts +++ b/src/backends/native-project-restart-preflight.ts @@ -6,6 +6,10 @@ import { ClosedCliError } from "../lib/cli-result.ts"; import { isRecord } from "../lib/guards.ts"; import { adaptNativeAwsEnvironment } from "./native-aws-environment.ts"; import { prepareNativeProjectAdaptation } from "./native-project-adaptation.ts"; +import { + admitNativeProjectBuilds, + nativeBuildsEnabled, +} from "./native-project-builds.ts"; import { discoverNativeHostDependency, type NativeHostDependency, @@ -288,6 +292,11 @@ async function pinRestartImages(opts: { spec.image = retainedImage; continue; } + if (spec.image === undefined && spec.build !== undefined) { + throw new ClosedCliError( + `Native restart reuses built images and never builds; no running or retained image was found for ${name}. Run hack up to build it; nothing was stopped.` + ); + } const image = String(spec.image); if (IMAGE.test(image)) { continue; @@ -390,9 +399,25 @@ export async function preflightNativeRestart(opts: { }) : undefined; input = metadata?.input ?? input; + // Restart never builds: an admitted build service must resolve to the image the + // current or retained graph already runs (see pinRestartImages). const specs = prepareNativeProjectServices( input, - opts.dependencyFile !== undefined + opts.dependencyFile !== undefined, + new Set( + admitNativeProjectBuilds({ + services: Object.fromEntries( + Object.entries( + JSON.parse(input.normalizedComposeJson).services ?? {} + ).filter((entry): entry is [string, Record] => + isRecord(entry[1]) + ) + ), + projectRoot: opts.scope.projectRoot, + composeFile: opts.composeFile, + enabled: nativeBuildsEnabled(), + }).keys() + ) ); const dependencies = stoppedRetry ? await readNativeRestartDependencyIntent({ diff --git a/src/backends/native-project-start.ts b/src/backends/native-project-start.ts index 57a3d7105..d9e89b982 100644 --- a/src/backends/native-project-start.ts +++ b/src/backends/native-project-start.ts @@ -20,6 +20,14 @@ import { preparedBaseArguments, } from "./native-prepared-base.ts"; import { prepareNativeProjectAdaptation } from "./native-project-adaptation.ts"; +import { + admitNativeProjectBuilds, + buildNativeProjectImages, + invokeNativeBuild, + type NativeBuildRequest, + nativeBuildsEnabled, + nativeRebuildRequested, +} from "./native-project-builds.ts"; import { nativeSharedSourceFlags, nativeStartCacheSource, @@ -258,6 +266,7 @@ type Dependencies = { recoverHttps: typeof recoverNativeHttpsLease; cleanupTemporary: (directory: string) => Promise; invoke: typeof invokeNativeRuntime; + build: typeof invokeNativeBuild; load: typeof loadNativeProjectRun; loadRestart: typeof loadNativeRestartIntent; save: typeof saveNativeProjectRun; @@ -279,6 +288,7 @@ const DEFAULTS: Dependencies = { cleanupTemporary: (directory) => rm(directory, { recursive: true, force: true }), invoke: invokeNativeRuntime, + build: invokeNativeBuild, load: loadNativeProjectRun, loadRestart: loadNativeRestartIntent, save: saveNativeProjectRun, @@ -328,9 +338,59 @@ function requireActiveStartup(signal: AbortSignal | undefined): void { throw canceled(); } } +function admitStartBuilds( + input: NativeProjectInput, + opts: { + readonly scope: { readonly projectRoot: string }; + readonly composeFile: string; + } +): ReadonlyMap { + return admitNativeProjectBuilds({ + services: composeServices(input), + projectRoot: opts.scope.projectRoot, + composeFile: opts.composeFile, + enabled: nativeBuildsEnabled(), + }); +} + +/** A service names an image, or is an admitted build that names none yet. */ +function imageOrAdmittedBuild( + name: string, + value: Readonly>, + builds: ReadonlySet +): boolean { + if (value.build === undefined) { + return typeof value.image === "string"; + } + return builds.has(name) && value.image === undefined; +} + +/** The normalized Compose services, or the closed configuration refusal. */ +function composeServices( + input: NativeProjectInput +): Record> { + const compose: unknown = JSON.parse(input.normalizedComposeJson); + if (!(isRecord(compose) && isRecord(compose.services))) { + throw refused(); + } + const services: Record> = {}; + for (const [name, value] of Object.entries(compose.services)) { + if (!isRecord(value)) { + throw refused(); + } + services[name] = value; + } + return services; +} + +/** + * `builds` names the services whose Compose `build` was admitted + * (`admitNativeProjectBuilds`); only those may omit `image` until they are built. + */ export function prepareNativeProjectServices( input: NativeProjectInput, - hasHostDependencies = false + hasHostDependencies = false, + builds: ReadonlySet = new Set() ): Record> { const compose: unknown = JSON.parse(input.normalizedComposeJson); if (!(isRecord(compose) && isRecord(compose.services))) { @@ -339,9 +399,7 @@ export function prepareNativeProjectServices( const result: Record> = {}; for (const [name, value] of Object.entries(compose.services)) { if ( - !isRecord(value) || - typeof value.image !== "string" || - value.build !== undefined || + !(isRecord(value) && imageOrAdmittedBuild(name, value, builds)) || (value.extra_hosts !== undefined && !hasHostDependencies) || value.ports !== undefined || value.logging !== undefined || @@ -1010,9 +1068,13 @@ export async function startNativeProject(opts: { phase: "before-runtime", }) ).input; + // Pure build admission before hooks, VM or provider start; refuses when disabled. + const builds = admitStartBuilds(input, opts); + const buildServices = new Set(builds.keys()); let specs = prepareNativeProjectServices( input, - opts.dependencyFile !== undefined + opts.dependencyFile !== undefined, + buildServices ); // Refuse malformed or unacquirable references before hooks, VM or provider start. await admitNativeProjectImages({ @@ -1065,7 +1127,8 @@ export async function startNativeProject(opts: { input = (await deps.adaptAws({ input, ...selection.aws })).input; specs = prepareNativeProjectServices( input, - opts.dependencyFile !== undefined + opts.dependencyFile !== undefined, + buildServices ); } const dependencyDiscoveryDeadline = performance.now() + 60_000; @@ -1159,7 +1222,8 @@ export async function startNativeProject(opts: { requireActiveStartup(controller.signal); specs = prepareNativeProjectServices( input, - opts.dependencyFile !== undefined + opts.dependencyFile !== undefined, + buildServices ); prepareNativeDependencyServices({ dependencies: hostDependencies, @@ -1172,6 +1236,18 @@ export async function startNativeProject(opts: { restore, invoke: deps.invoke, }); + await buildNativeProjectImages({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + branch: opts.scope.branch, + builds, + specs, + retainedImages, + restore: Boolean(restore), + rebuild: nativeRebuildRequested(), + signal: controller.signal, + invoke: deps.build, + }); await pinNativeProjectImages({ runtime: opts.runtime, projectRoot: opts.scope.projectRoot, diff --git a/tests/native-compose-backend-features-cli.test.ts b/tests/native-compose-backend-features-cli.test.ts index 34c16bb70..0ef0e5a26 100644 --- a/tests/native-compose-backend-features-cli.test.ts +++ b/tests/native-compose-backend-features-cli.test.ts @@ -10,6 +10,7 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, relative, resolve } from "node:path"; +import { NATIVE_BUILDS_DISABLED_REFUSAL } from "../src/backends/native-project-builds.ts"; import { NATIVE_CONFIGURATION_REFUSAL } from "../src/backends/native-project-start.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; @@ -325,16 +326,17 @@ test( test( "control: ordinary labels and a basic build keep today's native refusal", async () => { - for (const service of [ - { image: IMAGE, labels: { "com.example.team": "web" } }, - { build: ".." }, - ]) { + for (const [service, refusal] of [ + [ + { image: IMAGE, labels: { "com.example.team": "web" } }, + NATIVE_CONFIGURATION_REFUSAL, + ], + // A basic build stays refused unless native builds are explicitly enabled. + [{ build: ".." }, NATIVE_BUILDS_DISABLED_REFUSAL], + ] as const) { const f = await fixture({ services: { web: service } }); const result = await cli(f, ["up"]); - expectOneClosedLine( - result, - `E_NATIVE_PROJECT_UNSUPPORTED: ${NATIVE_CONFIGURATION_REFUSAL}` - ); + expectOneClosedLine(result, `E_NATIVE_PROJECT_UNSUPPORTED: ${refusal}`); expect(result.output).not.toContain("HACK_RUNTIME_BACKEND=compose"); // The check passed: native up went on to its own source preflight. expect(await f.calls()).toEqual(["runtime check-project-share"]); diff --git a/tests/native-project-builds.test.ts b/tests/native-project-builds.test.ts new file mode 100644 index 000000000..a77e5c060 --- /dev/null +++ b/tests/native-project-builds.test.ts @@ -0,0 +1,331 @@ +import { afterEach, expect, test } from "bun:test"; +import { + chmod, + mkdir, + mkdtemp, + readFile, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + admitNativeProjectBuild, + admitNativeProjectBuilds, + buildNativeProjectImages, + invokeNativeBuild, + NATIVE_BUILDS_DISABLED_REFUSAL, + type NativeBuildRequest, + nativeBuildsEnabled, +} from "../src/backends/native-project-builds.ts"; +import { NativeRuntimeRequestError } from "../src/backends/native-runtime-client.ts"; +import { HackCliError } from "../src/lib/cli-result.ts"; + +const IMAGE = `sha256:${"a".repeat(64)}`; +const RETAINED = `sha256:${"b".repeat(64)}`; +const KEY = "c".repeat(64); +const VALUE = "synthetic-literal-value"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }); + } +}); +async function root(): Promise { + const path = await mkdtemp(join(tmpdir(), "hack-native-builds-")); + roots.push(path); + return path; +} + +function admit( + service: Record, + composeFile = "/p/.hack/compose.yaml" +) { + return admitNativeProjectBuild({ service, projectRoot: "/p", composeFile }); +} + +test("admitted build forms follow Compose paths and literal arguments", () => { + expect(admit({ build: ".." })).toEqual({ + context: ".", + dockerfile: "Dockerfile", + target: null, + args: {}, + }); + expect( + admit({ + build: { + context: "../app", + dockerfile: "docker/Dockerfile.dev", + target: "base", + args: { VERSION: "1.2", RETRIES: 3 }, + }, + }) + ).toEqual({ + context: "app", + dockerfile: "docker/Dockerfile.dev", + target: "base", + args: { VERSION: "1.2", RETRIES: "3" }, + }); + expect(admit({ build: { args: ["A=1", "B=x=y"] } })?.args).toEqual({ + A: "1", + B: "x=y", + }); + expect(admit({ image: "busybox" })).toBeUndefined(); +}); + +test.each([ + [{ image: "busybox", build: "." }, "both image and build"], + [{ build: { context: ".", ssh: ["default"] } }, "only build context"], + [ + { build: { context: ".", platforms: ["linux/arm64"] } }, + "only build context", + ], + [{ build: { args: { A: "${HOST_VALUE}" } } }, "interpolation"], + [{ build: { args: { A: null } } }, "interpolation and null"], + [{ build: { args: ["INHERITED"] } }, "KEY=VALUE"], + [{ build: { args: { "bad-name": "x" } } }, "unique identifiers"], + [{ build: "$CONTEXT" }, "interpolation"], + [{ build: "../.." }, "inside the project"], + [{ build: "https://example.invalid/repo.git" }, "remote build contexts"], + [ + { build: { dockerfile: "/etc/Dockerfile" } }, + "relative to the build context", + ], + [{ build: { target: "-x" } }, "stage name"], + [{ build: 7 }, "context path or a mapping"], +])("unsupported build %j refuses before any effect", (service, text) => { + const error = (() => { + try { + admit(service); + } catch (caught: unknown) { + return caught; + } + })(); + expect(error).toBeInstanceOf(HackCliError); + expect(error).toMatchObject({ code: "E_NATIVE_PROJECT_UNSUPPORTED" }); + expect(String((error as Error).message)).toContain(text); + expect(String((error as Error).message)).not.toContain("HOST_VALUE"); +}); + +test("builds refuse unless explicitly enabled, and projects without builds are unaffected", () => { + const services = { web: { build: "." }, db: { image: "postgres:17" } }; + expect(() => + admitNativeProjectBuilds({ + services, + projectRoot: "/p", + composeFile: "/p/compose.yaml", + enabled: false, + }) + ).toThrow(NATIVE_BUILDS_DISABLED_REFUSAL); + expect( + admitNativeProjectBuilds({ + services: { db: { image: "postgres:17" } }, + projectRoot: "/p", + composeFile: "/p/compose.yaml", + enabled: false, + }).size + ).toBe(0); + expect([ + ...admitNativeProjectBuilds({ + services, + projectRoot: "/p", + composeFile: "/p/compose.yaml", + enabled: true, + }).keys(), + ]).toEqual(["web"]); + expect(nativeBuildsEnabled({ HACK_NATIVE_BUILDS: "1" })).toBe(true); + expect(nativeBuildsEnabled({ HACK_NATIVE_BUILDS: "true" })).toBe(false); +}); + +const runtime = { + binary: "/unused/hack-native", + home: "/unused/home", +} as never; +const request = ( + args: Record = { A: VALUE } +): NativeBuildRequest => ({ + context: "app", + dockerfile: "Dockerfile", + target: null, + args, +}); + +test("identical builds run once, pin every service and keep the request private", async () => { + const project = await root(); + const specs: Record> = { + one: { build: "app" }, + two: { build: "app" }, + other: { build: { context: "app", target: "dev" } }, + }; + const seen: { body: Record; mode: number; path: string }[] = + []; + await buildNativeProjectImages({ + runtime, + projectRoot: project, + branch: "feature", + builds: new Map([ + ["one", request()], + ["two", request()], + ["other", { ...request(), target: "dev" }], + ]), + specs, + retainedImages: new Map(), + restore: false, + rebuild: false, + signal: new AbortController().signal, + invoke: async ({ requestFile }) => { + seen.push({ + body: JSON.parse(await readFile(requestFile, "utf8")), + mode: (await stat(requestFile)).mode & 0o777, + path: requestFile, + }); + return { imageId: IMAGE, buildKey: KEY, reused: false }; + }, + }); + expect(seen).toHaveLength(2); + expect(seen[0]?.mode).toBe(0o600); + expect(seen[0]?.body).toMatchObject({ + version: 1, + branch: "feature", + context: "app", + policy: "missing", + args: { A: VALUE }, + }); + expect(seen[1]?.body.target).toBe("dev"); + for (const { path } of seen) { + expect(await Bun.file(path).exists()).toBe(false); + } + for (const spec of Object.values(specs)) { + expect(spec).toEqual({ image: IMAGE }); + } +}); + +test("retained images win, and restore never builds a missing one", async () => { + const project = await root(); + const calls: string[] = []; + const invoke = async () => { + calls.push("build"); + return { imageId: IMAGE, buildKey: KEY, reused: true }; + }; + const base = { + runtime, + projectRoot: project, + branch: null, + rebuild: false, + signal: new AbortController().signal, + invoke, + }; + const specs = { web: { build: "app" } as Record }; + await buildNativeProjectImages({ + ...base, + builds: new Map([["web", request()]]), + specs, + retainedImages: new Map([["web", RETAINED]]), + restore: true, + }); + expect(specs.web).toEqual({ image: RETAINED }); + expect(calls).toEqual([]); + + const missing = await buildNativeProjectImages({ + ...base, + builds: new Map([["worker", request()]]), + specs: { worker: { build: "app" } }, + retainedImages: new Map([["web", RETAINED]]), + restore: true, + }).catch((error: unknown) => error); + expect(String(missing)).toContain("Nothing was built"); + expect(calls).toEqual([]); + + // A retained run whose configuration changed selects no retained images and builds. + await buildNativeProjectImages({ + ...base, + builds: new Map([["worker", request()]]), + specs: { worker: { build: "app" } }, + retainedImages: new Map(), + restore: true, + }); + expect(calls).toEqual(["build"]); +}); + +test("an explicit rebuild asks the runtime to build again", async () => { + const project = await root(); + let policy: unknown; + await buildNativeProjectImages({ + runtime, + projectRoot: project, + branch: null, + builds: new Map([["web", request()]]), + specs: { web: { build: "app" } }, + retainedImages: new Map(), + restore: false, + rebuild: true, + signal: new AbortController().signal, + invoke: async ({ requestFile }) => { + policy = JSON.parse(await readFile(requestFile, "utf8")).policy; + return { imageId: IMAGE, buildKey: KEY, reused: false }; + }, + }); + expect(policy).toBe("rebuild"); +}); + +async function fakeRuntime( + script: string +): Promise<{ binary: string; home: string }> { + const directory = await root(); + const binary = join(directory, "hack-native"); + await writeFile(binary, `#!/bin/sh\n${script}\n`, { mode: 0o700 }); + await chmod(binary, 0o700); + const home = join(directory, "home"); + await mkdir(home); + return { binary, home }; +} + +test("build output streams through while the runtime's closed error is returned", async () => { + const fake = await fakeRuntime( + `printf 'step 1\\nstep 2\\n' >&2\nprintf '{"code":"native_build_failed","message":"The image build failed."}\\n' >&2\nexit 2` + ); + const written: string[] = []; + const error = await invokeNativeBuild({ + runtime: fake as never, + cwd: fake.home, + requestFile: join(fake.home, "request.json"), + write: (text) => written.push(text), + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(NativeRuntimeRequestError); + expect(error).toMatchObject({ nativeCode: "native_build_failed" }); + expect(written.join("")).toBe("step 1\nstep 2\n"); + + const ok = await fakeRuntime( + `printf 'progress\\n' >&2\nprintf '{"image_id":"${IMAGE}","build_key":"${KEY}","tag":"t","reused":false,"base_pins":{}}'` + ); + const built = await invokeNativeBuild({ + runtime: ok as never, + cwd: ok.home, + requestFile: join(ok.home, "request.json"), + write: () => undefined, + }); + expect(built).toEqual({ imageId: IMAGE, buildKey: KEY, reused: false }); +}); + +test("cancellation sends SIGTERM so the runtime can stop its build client", async () => { + const fake = await fakeRuntime( + `trap 'printf terminated > "$0.signal"; exit 143' TERM\nprintf 'started\\n' >&2\nwhile :; do sleep 0.05; done` + ); + const controller = new AbortController(); + const pending = invokeNativeBuild({ + runtime: fake as never, + cwd: fake.home, + requestFile: join(fake.home, "request.json"), + signal: controller.signal, + write: (text) => { + if (text.includes("started")) { + controller.abort(); + } + }, + }).catch((caught: unknown) => caught); + const error = await pending; + expect(String(error)).toContain("canceled during an image build"); + expect(await readFile(`${fake.binary}.signal`, "utf8")).toBe("terminated"); +}); From 297df5a0edea31891d9ed2781f5e81570632c3bb Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:14:41 -0500 Subject: [PATCH 2/5] fix(native): drop the build declaration when restart pins a built image Restart now removes a service's build declaration when it adopts the running or retained content ID, matching startup's build pinning, so an unchanged project's restart review equals the saved image-only plan instead of tripping the source compatibility check. Co-Authored-By: Claude Opus 5.5 --- docs/guides/native-candidate.md | 6 +- .../native-project-restart-preflight.ts | 3 + tests/native-project-restart.test.ts | 77 +++++++++++++++++++ 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index ab743ccfd..8a8fd1d97 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -243,9 +243,9 @@ With it: build keys (`secrets`, `ssh`, `cache_from`, `platforms`, `network` and the rest), remote contexts and `image` together with `build` refuse. Values are never shown. - After the runtime is up, each distinct build runs once with - `runtime build-image` (below), and its output streams to your terminal. Every - service with that build is then pinned to the built content ID, so review, readiness - and restore treat it like an image service. + `runtime build-image` (below), and its filtered output streams to your terminal. + Every service with that build is then pinned to the built content ID, so review, + readiness and restore treat it like an image service. - An existing image for the same build inputs is reused without building. Set `HACK_NATIVE_REBUILD=1` to build again; base-image tags then resolve again too. Edits to other files in the context do not trigger a rebuild, as with Compose. diff --git a/src/backends/native-project-restart-preflight.ts b/src/backends/native-project-restart-preflight.ts index 550881f8d..53309f28b 100644 --- a/src/backends/native-project-restart-preflight.ts +++ b/src/backends/native-project-restart-preflight.ts @@ -290,6 +290,9 @@ async function pinRestartImages(opts: { const retainedImage = opts.retainedImages.get(name); if (retainedImage) { spec.image = retainedImage; + // Startup pins a built image the same way: the content ID replaces the build + // declaration, so the restart review matches the saved image-only plan. + Reflect.deleteProperty(spec, "build"); continue; } if (spec.image === undefined && spec.build !== undefined) { diff --git a/tests/native-project-restart.test.ts b/tests/native-project-restart.test.ts index c6450adbe..c0f2452db 100644 --- a/tests/native-project-restart.test.ts +++ b/tests/native-project-restart.test.ts @@ -947,6 +947,83 @@ test("stopped recovery reviews retained image IDs and resolves tags only after o } }); +test("restart pins a build service to its retained image and drops the build declaration", async () => { + const listener = await listenerIntentFixture(); + const previous = process.env.HACK_NATIVE_BUILDS; + process.env.HACK_NATIVE_BUILDS = "1"; + try { + let reviewed: Record | undefined; + const builtImage = `sha256:${"7".repeat(64)}`; + await preflightNativeRestart({ + ...listener.options, + recoverStopped: true, + dependencies: { + ...listener.options.dependencies, + retainedImages: selectNativeRetainedImages, + prepare: async () => ({ + originalSha256: "1".repeat(64), + environmentFiles: [], + serviceNames: ["app"], + normalizedComposeJson: JSON.stringify({ + services: { app: { build: { context: "..", args: { A: "1" } } } }, + }), + managedEnvironment: {}, + lifecycleHostEnvironment: {}, + effectiveEnvName: "qa", + }), + invoke: async (options) => { + if (options.args[1] === "inspect") { + const observed = stoppedRetainedGraph(); + return { + ...observed, + receipt: { + ...observed.receipt, + normalized_input: { + namespace: run.namespace, + original_compose_sha256: "1".repeat(64), + normalized_compose_sha256: "3".repeat(64), + }, + resources: { + default: observed.receipt.resources.default, + data: observed.receipt.resources.data, + "container:app": { + kind: "container", + key: "app", + image: builtImage, + }, + }, + }, + }; + } + if (options.args[1] === "ensure-image") { + throw new Error("restart must not resolve or build images"); + } + return await listener.options.dependencies!.invoke!(options); + }, + review: async (review) => { + reviewed = JSON.parse(review.input.normalizedComposeJson).services + .app; + return await review.run({ + planId: run.planId, + namespace: run.namespace, + report: {}, + projectArgs: [], + }); + }, + }, + }); + expect(reviewed?.image).toBe(builtImage); + expect(reviewed).not.toHaveProperty("build"); + } finally { + if (previous === undefined) { + Reflect.deleteProperty(process.env, "HACK_NATIVE_BUILDS"); + } else { + process.env.HACK_NATIVE_BUILDS = previous; + } + await rm(listener.directory, { recursive: true, force: true }); + } +}); + test("invalid retained image provenance refuses stopped preflight before tag resolution", async () => { const listener = await listenerIntentFixture(); try { From 1d42589481aaffa97dde165e9b33df8cdd27c858 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:24:30 -0500 Subject: [PATCH 3/5] fix(native): keep build arguments named __proto__ Admitted build arguments now collect into a null-prototype dictionary, so an argument named __proto__ is an own property that reaches the runtime request and the build identity instead of invoking the inherited prototype setter. Co-Authored-By: Claude Opus 5.5 --- src/backends/native-project-builds.ts | 3 ++- tests/native-project-builds.test.ts | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/backends/native-project-builds.ts b/src/backends/native-project-builds.ts index 369b65e2a..63bf3965d 100644 --- a/src/backends/native-project-builds.ts +++ b/src/backends/native-project-builds.ts @@ -103,7 +103,8 @@ function buildArguments(value: unknown): Record { if (entries.length > MAX_ARGS) { throw refused("a native build accepts at most 64 build args."); } - const result: Record = {}; + // A null prototype keeps every admitted name an own property, `__proto__` included. + const result: Record = Object.create(null); for (const [name, raw] of entries) { if (!ARGUMENT.test(name) || Object.hasOwn(result, name)) { throw refused("build arg names must be unique identifiers."); diff --git a/tests/native-project-builds.test.ts b/tests/native-project-builds.test.ts index a77e5c060..e09f01e3a 100644 --- a/tests/native-project-builds.test.ts +++ b/tests/native-project-builds.test.ts @@ -75,6 +75,31 @@ test("admitted build forms follow Compose paths and literal arguments", () => { expect(admit({ image: "busybox" })).toBeUndefined(); }); +test("an argument named __proto__ survives admission and serialization", () => { + const requested = '{"__proto__":"requested-value"}'; + for (const args of [["__proto__=requested-value"], JSON.parse(requested)]) { + const request = admit({ build: { args } }); + expect(Object.keys(request?.args ?? {})).toEqual(["__proto__"]); + expect(JSON.stringify(request?.args)).toBe(requested); + } + // Declarations that differ only in that argument stay distinct builds. + const builds = admitNativeProjectBuilds({ + services: { + plain: { build: { context: "..", args: ["A=1"] } }, + proto: { + build: { context: "..", args: ["A=1", "__proto__=requested-value"] }, + }, + }, + projectRoot: "/p", + composeFile: "/p/.hack/compose.yaml", + enabled: true, + }); + expect(JSON.stringify(builds.get("proto")?.args)).toBe( + '{"A":"1","__proto__":"requested-value"}' + ); + expect(JSON.stringify(builds.get("plain")?.args)).toBe('{"A":"1"}'); +}); + test.each([ [{ image: "busybox", build: "." }, "both image and build"], [{ build: { context: ".", ssh: ["default"] } }, "only build context"], From b007be2475513650ad6720b0ed5cdbc3a5691a28 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:52:27 -0500 Subject: [PATCH 4/5] fix(native): build only active profiles and refuse managed secrets as build args Native up now follows Compose up: only build services in the selected profiles are built and pinned. An inactive build service keeps its build declaration, which review accepts for inactive services, so restore and restart need no image for it; selecting its profile in a later fresh up builds it then. Build args are not secrets, so a build arg whose value equals a hack env secret of the project and environment now refuses before any hook or VM effect, naming only the argument. The comparison is in memory and no value is shown. Co-Authored-By: Claude Opus 5.5 --- docs/guides/native-candidate.md | 12 +- src/backends/native-project-builds.ts | 61 +++++++++- src/backends/native-project-input.ts | 38 +++++++ .../native-project-restart-preflight.ts | 8 ++ src/backends/native-project-start.ts | 13 ++- tests/native-project-builds.test.ts | 106 ++++++++++++++++++ tests/native-project-input.test.ts | 36 ++++++ tests/native-project-restart.test.ts | 17 ++- 8 files changed, 280 insertions(+), 11 deletions(-) diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 836023efd..b566ddfde 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -242,6 +242,10 @@ With it: interpolation, a list item without `=` (host variable inheritance), `null`, other build keys (`secrets`, `ssh`, `cache_from`, `platforms`, `network` and the rest), remote contexts and `image` together with `build` refuse. Values are never shown. +- Build args are not secrets: as with Docker, the image records them in its history. + A build arg whose value equals a `hack env` secret value of this project and + environment refuses before any hook or VM effect, naming only the argument. Pass + secrets with BuildKit build secrets instead; native builds do not support them yet. - After the runtime is up, each distinct build runs once with `runtime build-image` (below), and its filtered output streams to your terminal. Every service with that build is then pinned to the built content ID, so review, @@ -252,9 +256,11 @@ With it: - Restoring an unchanged retained run uses the retained images and never builds; if one is missing, `up` refuses and asks you to start fresh. `hack restart` also never builds: it reuses the running or retained image, or refuses before stopping anything. -- Services in profiles you did not select are built too, because the native review - pins every declared service. Native builds are `linux/arm64` only; - `platform: linux/amd64` is still refused. +- As with Compose `up`, only build services in the selected profiles are built and + pinned. A build service in a profile you did not select is not built, and restore + and restart need no image for it; selecting its profile in a later fresh `up` builds + it then. Native builds are `linux/arm64` only; `platform: linux/amd64` is still + refused. `hack doctor` reports the same features for the current project in its `compose backend features` check. With `--json`, the check's `composeBackend.features` diff --git a/src/backends/native-project-builds.ts b/src/backends/native-project-builds.ts index 63bf3965d..0d1b737eb 100644 --- a/src/backends/native-project-builds.ts +++ b/src/backends/native-project-builds.ts @@ -421,6 +421,53 @@ function systemEnvironment(): Record { return env; } +/** + * Whether a service runs for the selected profiles, as in Compose `up`: a service with + * no `profiles` always runs; otherwise one of its profiles must be selected. A + * malformed `profiles` field counts as active, so it is built and reviewed. + */ +export function nativeServiceActive( + service: Readonly>, + profiles: readonly string[] +): boolean { + const declared = service.profiles; + if (declared === undefined) { + return true; + } + if (!Array.isArray(declared) || declared.length === 0) { + return true; + } + return declared.some( + (profile) => typeof profile === "string" && profiles.includes(profile) + ); +} + +/** + * Refuse, before any hook, VM or build effect, a build argument whose value equals a + * secret value Hack manages for this project and environment. Build arguments are + * not secrets (an image records them in its history), so a managed secret must not + * travel as one. The comparison is in memory; no value is ever shown. + */ +export function refuseManagedSecretBuildArguments(opts: { + readonly builds: ReadonlyMap; + readonly managedSecretValues: readonly string[]; +}): void { + if (opts.managedSecretValues.length === 0) { + return; + } + const secrets = new Set(opts.managedSecretValues.filter((v) => v !== "")); + for (const [service, request] of opts.builds) { + for (const [name, value] of Object.entries(request.args)) { + if (secrets.has(value)) { + throw new HackCliError({ + code: "E_NATIVE_PROJECT_UNSUPPORTED", + message: `Native foreground up cannot build ${service}: build arg ${name} equals a hack env secret value. Build args are not secrets (images record them); pass secrets with BuildKit build secrets instead, which native builds do not support yet. Values were omitted; no hook, VM or build was started.`, + }); + } + } + } +} + /** A refusal when a retained run's built image is gone: restore never builds. */ function retainedMissing(service: string): ClosedCliError { return new ClosedCliError( @@ -429,14 +476,18 @@ function retainedMissing(service: string): ClosedCliError { } /** - * Build (or reuse) every admitted build service and pin it to its content ID. - * Identical declarations build once. Retained content IDs always win, and a - * restore with a missing retained image refuses instead of building. + * Build (or reuse) every admitted build service in the selected profiles and pin it to + * its content ID. As with Compose `up`, a build service outside the selected profiles + * is neither built nor pinned: it keeps its `build` declaration, which review accepts + * for inactive services, so restore and restart never need an image for it. Identical + * declarations build once. Retained content IDs always win, and a restore with a + * missing retained image refuses instead of building. */ export async function buildNativeProjectImages(opts: { readonly runtime: NativeRuntimeSelection; readonly projectRoot: string; readonly branch: string | null | undefined; + readonly profiles: readonly string[]; readonly builds: ReadonlyMap; readonly specs: Record>; readonly retainedImages: ReadonlyMap; @@ -456,6 +507,7 @@ type PendingBuild = { request: NativeBuildRequest; services: string[] }; /** Pin retained images, refuse a restore without one, and group identical builds. */ function pendingBuilds(opts: { + readonly profiles: readonly string[]; readonly builds: ReadonlyMap; readonly specs: Record>; readonly retainedImages: ReadonlyMap; @@ -469,6 +521,9 @@ function pendingBuilds(opts: { "Native build services changed during startup; values omitted." ); } + if (!nativeServiceActive(spec, opts.profiles)) { + continue; + } const retained = opts.retainedImages.get(service); if (retained !== undefined) { pin(spec, retained); diff --git a/src/backends/native-project-input.ts b/src/backends/native-project-input.ts index ab83234d9..f14daaeea 100644 --- a/src/backends/native-project-input.ts +++ b/src/backends/native-project-input.ts @@ -19,6 +19,11 @@ export type NativeProjectInput = { readonly managedEnvironment: Readonly< Record>> >; + /** + * Private in-memory only: the values Hack stores as secrets for this project and + * environment, so build arguments can refuse to carry one. Never log or write it. + */ + readonly managedSecretValues?: readonly string[]; /** Private lifecycle environment; not container environment. */ readonly lifecycleHostEnvironment: Readonly>; readonly effectiveEnvName: string | null; @@ -131,6 +136,38 @@ function normalizeManagedEnvironment( return managedEnvironment; } +/** Every value the resolved project env stores as a secret, for any scope or target. */ +function managedSecretValues( + resolved: Awaited> +): string[] { + if (!resolved) { + return []; + } + const values = new Set(); + const collect = ( + metadata: typeof resolved.effectiveMetadata, + valuesFor: (target: string) => Readonly> | undefined + ) => { + for (const [target, keys] of Object.entries(metadata)) { + const resolvedValues = valuesFor(target); + for (const [key, entry] of Object.entries(keys)) { + const value = resolvedValues?.[key]; + if (entry.secret && typeof value === "string" && value !== "") { + values.add(value); + } + } + } + }; + collect(resolved.effectiveMetadata, (target) => + target === "global" ? resolved.globalEnv : resolved.serviceEnv[target] + ); + collect( + resolved.hostEffectiveMetadata, + (target) => resolved.hostTargetEnv[target] + ); + return [...values]; +} + /** * Prepare public normalization and separate private delivery without generating files. * Unrelated Compose fields remain intact for native capability admission to review; @@ -176,6 +213,7 @@ export async function prepareNativeProjectInput(opts: { originalSha256: createHash("sha256").update(bytes).digest("hex"), normalizedComposeJson, managedEnvironment, + managedSecretValues: managedSecretValues(resolved), lifecycleHostEnvironment: resolved ? selectProjectEnvValuesForExecutionTarget({ resolved, diff --git a/src/backends/native-project-restart-preflight.ts b/src/backends/native-project-restart-preflight.ts index 53309f28b..f095d6d4a 100644 --- a/src/backends/native-project-restart-preflight.ts +++ b/src/backends/native-project-restart-preflight.ts @@ -9,6 +9,7 @@ import { prepareNativeProjectAdaptation } from "./native-project-adaptation.ts"; import { admitNativeProjectBuilds, nativeBuildsEnabled, + nativeServiceActive, } from "./native-project-builds.ts"; import { discoverNativeHostDependency, @@ -282,6 +283,7 @@ async function stoppedRestartMode(opts: { async function pinRestartImages(opts: { readonly runtime: NativeRuntimeSelection; readonly projectRoot: string; + readonly profiles: readonly string[]; readonly specs: Record>; readonly retainedImages: ReadonlyMap; readonly invoke: typeof invokeNativeRuntime; @@ -296,6 +298,11 @@ async function pinRestartImages(opts: { continue; } if (spec.image === undefined && spec.build !== undefined) { + // Outside the selected profiles a build service is never built or pinned + // (as at startup), so it keeps its build declaration. + if (!nativeServiceActive(spec, opts.profiles)) { + continue; + } throw new ClosedCliError( `Native restart reuses built images and never builds; no running or retained image was found for ${name}. Run hack up to build it; nothing was stopped.` ); @@ -461,6 +468,7 @@ export async function preflightNativeRestart(opts: { await pinRestartImages({ runtime: opts.runtime, projectRoot: opts.scope.projectRoot, + profiles: selected.profiles ?? [], specs, retainedImages, invoke: deps.invoke, diff --git a/src/backends/native-project-start.ts b/src/backends/native-project-start.ts index d9e89b982..b9da95e7b 100644 --- a/src/backends/native-project-start.ts +++ b/src/backends/native-project-start.ts @@ -27,6 +27,7 @@ import { type NativeBuildRequest, nativeBuildsEnabled, nativeRebuildRequested, + refuseManagedSecretBuildArguments, } from "./native-project-builds.ts"; import { nativeSharedSourceFlags, @@ -345,12 +346,17 @@ function admitStartBuilds( readonly composeFile: string; } ): ReadonlyMap { - return admitNativeProjectBuilds({ + const builds = admitNativeProjectBuilds({ services: composeServices(input), projectRoot: opts.scope.projectRoot, composeFile: opts.composeFile, enabled: nativeBuildsEnabled(), }); + refuseManagedSecretBuildArguments({ + builds, + managedSecretValues: input.managedSecretValues ?? [], + }); + return builds; } /** A service names an image, or is an admitted build that names none yet. */ @@ -433,6 +439,10 @@ async function pinNativeProjectImages(opts: { spec.image = retainedImage; continue; } + // A build service left unbuilt here is outside the selected profiles. + if (spec.image === undefined && spec.build !== undefined) { + continue; + } const image = String(spec.image); if (IMAGE.test(image)) { continue; @@ -1240,6 +1250,7 @@ export async function startNativeProject(opts: { runtime: opts.runtime, projectRoot: opts.scope.projectRoot, branch: opts.scope.branch, + profiles, builds, specs, retainedImages, diff --git a/tests/native-project-builds.test.ts b/tests/native-project-builds.test.ts index e09f01e3a..c281fb319 100644 --- a/tests/native-project-builds.test.ts +++ b/tests/native-project-builds.test.ts @@ -18,6 +18,8 @@ import { NATIVE_BUILDS_DISABLED_REFUSAL, type NativeBuildRequest, nativeBuildsEnabled, + nativeServiceActive, + refuseManagedSecretBuildArguments, } from "../src/backends/native-project-builds.ts"; import { NativeRuntimeRequestError } from "../src/backends/native-runtime-client.ts"; import { HackCliError } from "../src/lib/cli-result.ts"; @@ -190,6 +192,7 @@ test("identical builds run once, pin every service and keep the request private" runtime, projectRoot: project, branch: "feature", + profiles: [], builds: new Map([ ["one", request()], ["two", request()], @@ -238,6 +241,7 @@ test("retained images win, and restore never builds a missing one", async () => runtime, projectRoot: project, branch: null, + profiles: [], rebuild: false, signal: new AbortController().signal, invoke, @@ -274,6 +278,107 @@ test("retained images win, and restore never builds a missing one", async () => expect(calls).toEqual(["build"]); }); +test("only build services in the selected profiles are built and pinned, as with Compose up", async () => { + const project = await root(); + const built: string[] = []; + const invoke = async ({ requestFile }: { requestFile: string }) => { + built.push(JSON.parse(await readFile(requestFile, "utf8")).target ?? ""); + return { imageId: IMAGE, buildKey: KEY, reused: false }; + }; + const declared = () => ({ + web: { build: "app" } as Record, + tools: { build: "app", profiles: ["tools"] } as Record, + }); + const builds = new Map([ + ["web", request()], + ["tools", { ...request(), target: "tools" }], + ]); + const base = { + runtime, + projectRoot: project, + branch: null, + builds, + rebuild: false, + signal: new AbortController().signal, + invoke, + }; + const specs = declared(); + await buildNativeProjectImages({ + ...base, + profiles: [], + specs, + retainedImages: new Map(), + restore: false, + }); + expect(built).toEqual([""]); + expect(specs.web).toEqual({ image: IMAGE }); + // The inactive service keeps its declaration: no image, no build, no pin. + expect(specs.tools).toEqual({ build: "app", profiles: ["tools"] }); + + // Restoring that run needs no image for the inactive service. + built.length = 0; + const restored = declared(); + await buildNativeProjectImages({ + ...base, + profiles: [], + specs: restored, + retainedImages: new Map([["web", RETAINED]]), + restore: true, + }); + expect(built).toEqual([]); + expect(restored.web).toEqual({ image: RETAINED }); + expect(restored.tools).toEqual({ build: "app", profiles: ["tools"] }); + + // Selecting the profile later builds that service then. + const selected = declared(); + await buildNativeProjectImages({ + ...base, + profiles: ["tools"], + specs: selected, + retainedImages: new Map(), + restore: false, + }); + expect(built).toEqual(["", "tools"]); + expect(selected.tools).toEqual({ image: IMAGE, profiles: ["tools"] }); +}); + +test("service activity follows Compose profiles", () => { + expect(nativeServiceActive({}, [])).toBe(true); + expect(nativeServiceActive({ profiles: ["a"] }, [])).toBe(false); + expect(nativeServiceActive({ profiles: ["a", "b"] }, ["b"])).toBe(true); + expect(nativeServiceActive({ profiles: [] }, [])).toBe(true); + expect(nativeServiceActive({ profiles: "a" }, [])).toBe(true); +}); + +test("a build arg equal to a managed secret value refuses without showing it", () => { + const secret = "managed-secret-value-not-shown"; + const builds = new Map([ + ["web", request({ TOKEN: secret, MODE: "production" })], + ]); + let refusal: unknown; + try { + refuseManagedSecretBuildArguments({ + builds, + managedSecretValues: ["other", secret], + }); + } catch (error) { + refusal = error; + } + expect(refusal).toBeInstanceOf(HackCliError); + const error = refusal as HackCliError; + expect(error.code).toBe("E_NATIVE_PROJECT_UNSUPPORTED"); + expect(error.message).toContain( + "build arg TOKEN equals a hack env secret value" + ); + expect(error.message).not.toContain(secret); + // Values that are not managed secrets pass; so does any build with no secrets. + refuseManagedSecretBuildArguments({ + builds: new Map([["web", request({ MODE: "production" })]]), + managedSecretValues: [secret], + }); + refuseManagedSecretBuildArguments({ builds, managedSecretValues: [] }); +}); + test("an explicit rebuild asks the runtime to build again", async () => { const project = await root(); let policy: unknown; @@ -281,6 +386,7 @@ test("an explicit rebuild asks the runtime to build again", async () => { runtime, projectRoot: project, branch: null, + profiles: [], builds: new Map([["web", request()]]), specs: { web: { build: "app" } }, retainedImages: new Map(), diff --git a/tests/native-project-input.test.ts b/tests/native-project-input.test.ts index 8c04394d2..4df095a84 100644 --- a/tests/native-project-input.test.ts +++ b/tests/native-project-input.test.ts @@ -12,6 +12,8 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { prepareNativeProjectInput } from "../src/backends/native-project-input.ts"; +import { PROJECT_ENV_KEY_FILENAME } from "../src/constants.ts"; +import { setProjectEnvValue } from "../src/lib/project-env-config.ts"; import { restoreEnv } from "./helpers/env.ts"; import { fixtureGitEnv } from "./helpers/fixture-git.ts"; @@ -82,6 +84,40 @@ test("modern values stay private while original contracts and exact input hash s expect(await readFile(opts.composeFile)).toEqual(before); expect(await readdir(opts.projectDir)).toEqual(names); }); +test("only values stored as secrets are reported as managed secret values", async () => { + const opts = await fixture({ + services: { web: { image: "bun:1" }, worker: { image: "bun:1" } }, + }); + await writeFile( + join(opts.projectRoot, PROJECT_ENV_KEY_FILENAME), + "synthetic-key" + ); + const set = async ( + scope: string, + key: string, + value: string, + secret: boolean + ) => + await setProjectEnvValue({ + projectRoot: opts.projectRoot, + projectDir: opts.projectDir, + envName: null, + scope, + key, + value, + secret, + }); + await set("global", "TOKEN", "synthetic-global-secret", true); + await set("worker", "WORKER_KEY", "synthetic-worker-secret", true); + await set("web", "MODE", "production", false); + const result = await prepareNativeProjectInput(opts); + expect([...(result.managedSecretValues ?? [])].sort()).toEqual([ + "synthetic-global-secret", + "synthetic-worker-secret", + ]); + expect(result.normalizedComposeJson).not.toContain("synthetic-"); +}); + test("fourteen services and 110 managed variables each preserve overlay selection", async () => { const services = Object.fromEntries( Array.from({ length: 14 }, (_, i) => [`service${i}`, { image: "bun:1" }]) diff --git a/tests/native-project-restart.test.ts b/tests/native-project-restart.test.ts index c0f2452db..c5aefe0de 100644 --- a/tests/native-project-restart.test.ts +++ b/tests/native-project-restart.test.ts @@ -947,12 +947,13 @@ test("stopped recovery reviews retained image IDs and resolves tags only after o } }); -test("restart pins a build service to its retained image and drops the build declaration", async () => { +test("restart pins an active build service to its retained image, drops its build declaration and leaves inactive ones unbuilt", async () => { const listener = await listenerIntentFixture(); const previous = process.env.HACK_NATIVE_BUILDS; process.env.HACK_NATIVE_BUILDS = "1"; try { let reviewed: Record | undefined; + let inactive: Record | undefined; const builtImage = `sha256:${"7".repeat(64)}`; await preflightNativeRestart({ ...listener.options, @@ -965,7 +966,11 @@ test("restart pins a build service to its retained image and drops the build dec environmentFiles: [], serviceNames: ["app"], normalizedComposeJson: JSON.stringify({ - services: { app: { build: { context: "..", args: { A: "1" } } } }, + services: { + app: { build: { context: "..", args: { A: "1" } } }, + // Outside the run's profiles: never built, pinned or required. + tools: { build: "..", profiles: ["tools"] }, + }, }), managedEnvironment: {}, lifecycleHostEnvironment: {}, @@ -1001,8 +1006,11 @@ test("restart pins a build service to its retained image and drops the build dec return await listener.options.dependencies!.invoke!(options); }, review: async (review) => { - reviewed = JSON.parse(review.input.normalizedComposeJson).services - .app; + const services = JSON.parse( + review.input.normalizedComposeJson + ).services; + reviewed = services.app; + inactive = services.tools; return await review.run({ planId: run.planId, namespace: run.namespace, @@ -1014,6 +1022,7 @@ test("restart pins a build service to its retained image and drops the build dec }); expect(reviewed?.image).toBe(builtImage); expect(reviewed).not.toHaveProperty("build"); + expect(inactive).toEqual({ build: "..", profiles: ["tools"] }); } finally { if (previous === undefined) { Reflect.deleteProperty(process.env, "HACK_NATIVE_BUILDS"); From 7f42e328d9b3f95060a71a5db0d22778cb0c5c2f Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Fri, 6 Nov 2026 18:42:56 -0500 Subject: [PATCH 5/5] fix(native): admit native build fields and name services in build refusals (#628) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changes **HACK-1925: build args were refused before the builds path ran.** With `HACK_NATIVE_BUILDS=1`, the Compose-backend feature gate now admits exactly the build fields native builds implement: `context`, `dockerfile`, `target` and `args`. Both sides use one shared constant, `NATIVE_BUILD_FIELDS`. Native build admission then reviews the values of those fields. As a result: - Literal build args build natively. - A build arg whose value equals a managed `hack env` secret now reaches the existing refusal. That refusal names the service and the arg, never prints the value, and does not suggest the Compose backend (a Compose build would record the value in the image history too). - These still select the Compose backend: any other build field (`secrets`, `ssh`, `cache_from`, `platforms`, …) and `image` together with `build`. The refusal phrase names the native field set in this case. - Without native builds, today's basic-form classification and message are unchanged. - `hack doctor` applies the same classification, so doctor and `up` agree. **HACK-1927: build refusals now name the service.** - Admission refusals: `cannot build service : …`. - The builds-disabled refusal (now `nativeBuildsDisabledRefusal(service)`). - The secret-arg refusal. - Runtime failures from `runtime build-image`, wrapped in `runPendingBuilds`. A refusal with a build source policy code (`native_build_source`, `_dockerfile`, `_base_image`, `_argument`, `_reserved_label`, `_image_volumes`) says that it is the native build source policy and what to change. Other runtime failures name the service(s) and keep their message. The native codes are kept, and the code lookup uses only the map's own keys. - The candidate guide now says the pinned network-tool APKs come from Alpine v3.19 `main`/`aarch64`, and that v3.20 has files with the same names but different bytes. ## Validation - Focused tests (bounded runner, private HOME/TMPDIR, stubbed `docker`/`launchctl`/`security`): - `tests/native-project-builds.test.ts`, `tests/native-compose-backend-features.test.ts`, `tests/native-compose-backend-features-cli.test.ts`: 64 pass, 0 fail. - Related suites `native-project-restart`, `native-project-start`, `compose-startup-budget`, `doctor-json`, `doctor-command`: 157 pass, 0 fail. - New regression tests (synthetic values only): - Gate with builds enabled: implemented fields pass; unimplemented fields and image+build still refuse; disabled behavior is unchanged; the phrase follows the flag. - CLI with `HACK_NATIVE_BUILDS=1`: - A build arg passes the gate and reaches native start. - A build arg equal to a managed secret (set with `setProjectEnvValue`) refuses with one closed line naming the service and the arg. It does not show the value and does not mention `HACK_RUNTIME_BACKEND=compose`. The only runtime call is `check-project-share`. - An unimplemented field still gets the Compose-backend line. - Service-named admission refusals and the disabled refusal. - Runtime source-policy refusals, other runtime failures, and an inherited property name as a native code. - Checks: - `bun run --cwd packages/cli typecheck`: the only error is in `tests/fixtures/atomic-write-foreign-owner-worker.ts`. That file is unchanged, and the same error appears on the base head 8ec7460e99. - `bun run --cwd packages/cli check`: rc 0. - Biome on the changed files: clean. - `bun run docs:prepare && bun run docs:check`: pass. - `gitleaks protect --staged` and `gitleaks detect` over the range: no leaks. - Normal hooks. - Host check on an M5 VM. Isolated HOME, `HACK_DAEMON_AUTOSTART_DISABLE=1`, a fresh native home per fixture, and a bundle made of this branch's compiled CLI plus the base `hack-native`. The two `hack-native` files are byte-identical; this PR changes no Rust. - Fixture with a literal non-secret build arg: built in the VM, the service reached `ready-observed`, then stopped cleanly. - Fixture whose build arg equals a managed secret: refused with `cannot build service secretsvc: build arg GREETING equals a hack env secret value …`. The value is absent from the output, no Compose-backend suggestion is made, and the runtime stays `uninitialized`. - Fixture whose Dockerfile uses a non-Docker-Hub base: `cannot build service deniedsvc: the native build source policy refused it. Dockerfile line 1: … Use public Docker Hub images in the Dockerfile's FROM and COPY --from. No image was adopted.` - The full `bun test` suite was not run locally, as instructed for this unit. Hosted CI is the full-suite gate. ## Remaining gaps - The gate runs before managed env is loaded, so it cannot see secret values. Suppose a project has a Compose-only feature such as an external network, plus a build arg that carries a secret. It still gets the generic "use the Compose backend" line. The secret check only runs on the native builds path. - Value-level build problems such as a remote context or an absolute path no longer suggest the Compose backend when builds are enabled. Native build admission refuses them instead, naming the service. Refs HACK-1925 Refs HACK-1927 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- Prove it works Open in [code]smith · Need help on this PR? Tag @codesmith-bot with what you need. Co-authored-by: Dimitri Kennedy <344842+roodboi@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- docs/guides/native-candidate.md | 17 +- src/backends/native-project-builds.ts | 146 ++++++++++++++---- src/commands/doctor.ts | 5 + src/commands/project.ts | 4 + src/lib/doctor-compose-backend.ts | 13 +- src/lib/native-compose-backend-features.ts | 107 +++++++++---- ...ative-compose-backend-features-cli.test.ts | 86 ++++++++++- tests/native-compose-backend-features.test.ts | 58 +++++++ tests/native-project-builds.test.ts | 84 +++++++++- 9 files changed, 444 insertions(+), 76 deletions(-) diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 68e82a2d4..c37ca4587 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -192,7 +192,7 @@ Compose backend. Native support for them is planned after v5. | External network | A top-level network sets `external: true` or `external: {name: ...}`. A reviewed `x-hack-isolated: true` replacement is not external: it creates an owned internal bridge. A network that the `HACK_NATIVE_ADAPTATION` selection (see [Frontend integration status](#frontend-integration-status)) lists in `isolatedNetworks` gets the same replacement, so it is not external either. | | External volume | A top-level volume sets `external: true` or `external: {name: ...}`. The native runtime only creates volumes it owns, and there is no native replacement for an external volume. | | Reserved ownership label | A service, network or volume label key starts with `com.docker.compose.`, `io.hack-local.`, `io.hack.native-config.`, `io.hack.native-file.` or `io.hack.storage-witness.`. Compose, the native runtime and native-config Compose use these labels to prove which resources they own. | -| Build outside the basic form | A service `build` is not the basic form that [`hack config import --dry-run`](../reference/native-config-import.md#pure-basic-build-preview) accepts (a relative context and optional relative `dockerfile` and `target`), or the service also sets `image`. Build arguments, secrets, SSH, cache options, platforms and other build options are outside the basic form. | +| Build outside the basic form | A service `build` is not the basic form that [`hack config import --dry-run`](../reference/native-config-import.md#pure-basic-build-preview) accepts (a relative context and optional relative `dockerfile` and `target`), or the service also sets `image`. Build arguments, secrets, SSH, cache options, platforms and other build options are outside the basic form. With `HACK_NATIVE_BUILDS=1` (see below), only a build field other than `context`, `dockerfile`, `target` and `args`, or `image` together with `build`, counts; native build admission reviews the values of those four fields itself. | | Non-native platform | A service sets a `platform` other than `linux/arm64` or `linux/arm64/v8`, for example `linux/amd64`. The native VM runs Linux arm64 only. A missing or `null` `platform` is unset and passes. | Native `hack up` and `hack restart` check for these features after selecting the @@ -244,8 +244,14 @@ With it: remote contexts and `image` together with `build` refuse. Values are never shown. - Build args are not secrets: as with Docker, the image records them in its history. A build arg whose value equals a `hack env` secret value of this project and - environment refuses before any hook or VM effect, naming only the argument. Pass + environment refuses before any hook or VM effect, naming the service and the + argument but never the value. The refusal does not suggest the Compose backend, + because a Compose build would record the value in the image history too. Pass secrets with BuildKit build secrets instead; native builds do not support them yet. +- Every build refusal names the service. A refusal from the runtime's build source + policy (for example a base image that is not a public Docker Hub reference) says + that it is the build source policy and what to change. Other build failures name + the service and keep the runtime's message. - After the runtime is up, each distinct build runs once with `runtime build-image` (below), and its filtered output streams to your terminal. Every service with that build is then pinned to the built content ID, so review, @@ -548,8 +554,11 @@ saved run state. The report lists removed images and why each other one was kept Before startup, network-tool preparation additionally requires all four pinned Alpine aarch64 APKs in a current-user-owned private directory (mode `0700`), with -regular, singly linked mode-`0600` files. Acquire the matching public package bytes; -the command never downloads packages or accepts substitute versions: +regular, singly linked mode-`0600` files. The pins are the Alpine v3.19 `main` +repository for `aarch64`. Other Alpine releases, such as v3.20, publish files with +the same names but different bytes, which the command refuses. Acquire the matching +public package bytes; the command never downloads packages or accepts substitute +versions: | File | SHA-256 | | --- | --- | diff --git a/src/backends/native-project-builds.ts b/src/backends/native-project-builds.ts index 0d1b737eb..a7f60d13c 100644 --- a/src/backends/native-project-builds.ts +++ b/src/backends/native-project-builds.ts @@ -25,7 +25,18 @@ const IMAGE_ID = /^sha256:[a-f0-9]{64}$/; const BUILD_KEY = /^[a-f0-9]{64}$/; const ARGUMENT = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; const TARGET = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; -const BUILD_KEYS = new Set(["context", "dockerfile", "target", "args"]); +/** + * The Compose `build` fields native builds implement. The Compose-backend feature + * gate admits exactly these fields when builds are enabled, so a build it lets + * through reaches this module's own admission and secret check. + */ +export const NATIVE_BUILD_FIELDS = [ + "context", + "dockerfile", + "target", + "args", +] as const; +const BUILD_KEYS: ReadonlySet = new Set(NATIVE_BUILD_FIELDS); const MAX_ARGS = 64; /** The runtime's own budget is 30 minutes; the client waits a little longer. */ const BUILD_TIMEOUT_SECONDS = 30 * 60; @@ -57,36 +68,48 @@ export interface NativeBuildRequest { readonly args: Readonly>; } -/** Fixed refusal for a `build:` service while native builds are not enabled. */ -export const NATIVE_BUILDS_DISABLED_REFUSAL = - "Native foreground up builds Compose build services only with HACK_NATIVE_BUILDS=1; set it to build them in the native runtime. No hook, VM or build was started."; +/** The refusal for the first `build:` service while native builds are not enabled. */ +export function nativeBuildsDisabledRefusal(service: string): string { + return `Native foreground up cannot build service ${service}: it builds Compose build services only with HACK_NATIVE_BUILDS=1; set it to build them in the native runtime. No hook, VM or build was started.`; +} -function refused(detail: string): HackCliError { +/** "service a", or "services a and b", for build refusals. */ +function serviceList(services: readonly string[]): string { + if (services.length <= 1) { + return `service ${services[0] ?? ""}`; + } + return `services ${services.slice(0, -1).join(", ")} and ${services.at(-1)}`; +} + +/** The admission refusal for one service. Authored values are never included. */ +function refused(service: string, detail: string): HackCliError { return new HackCliError({ code: "E_NATIVE_PROJECT_UNSUPPORTED", - message: `Native foreground up cannot build this service: ${detail} Values were omitted; no hook, VM or build was started.`, + message: `Native foreground up cannot build service ${service}: ${detail} Values were omitted; no hook, VM or build was started.`, }); } -function literal(value: unknown, what: string): string { +function literal(service: string, value: unknown, what: string): string { if (typeof value !== "string" || value.length === 0 || value.includes("$")) { throw refused( + service, `${what} must be a literal non-empty string without Compose interpolation.` ); } return value; } -function argumentEntries(value: unknown): [string, unknown][] { +function argumentEntries(service: string, value: unknown): [string, unknown][] { if (isRecord(value)) { return Object.entries(value); } if (!Array.isArray(value)) { - throw refused("build args must be a mapping or a list."); + throw refused(service, "build args must be a mapping or a list."); } return value.map((item: unknown): [string, unknown] => { if (typeof item !== "string" || !item.includes("=")) { throw refused( + service, "list-form build args must be KEY=VALUE literals; inheriting a host variable is not supported." ); } @@ -95,25 +118,29 @@ function argumentEntries(value: unknown): [string, unknown][] { }); } -function buildArguments(value: unknown): Record { +function buildArguments( + service: string, + value: unknown +): Record { if (value === undefined) { return {}; } - const entries = argumentEntries(value); + const entries = argumentEntries(service, value); if (entries.length > MAX_ARGS) { - throw refused("a native build accepts at most 64 build args."); + throw refused(service, "a native build accepts at most 64 build args."); } // A null prototype keeps every admitted name an own property, `__proto__` included. const result: Record = Object.create(null); for (const [name, raw] of entries) { if (!ARGUMENT.test(name) || Object.hasOwn(result, name)) { - throw refused("build arg names must be unique identifiers."); + throw refused(service, "build arg names must be unique identifiers."); } const text = typeof raw === "number" && Number.isFinite(raw) ? String(raw) : raw; if (typeof text !== "string" || text.includes("$")) { // Env-derived values would be baked into image history. throw refused( + service, "build arg values must be literal strings or numbers; interpolation and null values are not supported." ); } @@ -123,41 +150,42 @@ function buildArguments(value: unknown): Record { } /** The declared build fields with Compose defaults, before path checks. */ -function buildFields(build: unknown): NativeBuildRequest { +function buildFields(service: string, build: unknown): NativeBuildRequest { if (typeof build === "string") { return { - context: literal(build, "the build context"), + context: literal(service, build, "the build context"), dockerfile: "Dockerfile", target: null, args: {}, }; } if (!isRecord(build)) { - throw refused("build must be a context path or a mapping."); + throw refused(service, "build must be a context path or a mapping."); } if (Object.keys(build).some((key) => !BUILD_KEYS.has(key))) { throw refused( + service, "only build context, dockerfile, target and literal args are supported." ); } const target = build.target === undefined ? null - : literal(build.target, "the build target"); + : literal(service, build.target, "the build target"); if (target !== null && !TARGET.test(target)) { - throw refused("the build target must be a stage name."); + throw refused(service, "the build target must be a stage name."); } return { context: build.context === undefined ? "." - : literal(build.context, "the build context"), + : literal(service, build.context, "the build context"), dockerfile: build.dockerfile === undefined ? "Dockerfile" - : literal(build.dockerfile, "the Dockerfile path"), + : literal(service, build.dockerfile, "the Dockerfile path"), target, - args: buildArguments(build.args), + args: buildArguments(service, build.args), }; } @@ -168,6 +196,8 @@ function buildFields(build: unknown): NativeBuildRequest { * runtime resolves it again without following symlinks. */ export function admitNativeProjectBuild(opts: { + /** The Compose service name; every refusal names it. */ + readonly name: string; readonly service: Readonly>; readonly projectRoot: string; readonly composeFile: string; @@ -176,21 +206,25 @@ export function admitNativeProjectBuild(opts: { if (build === undefined) { return; } + const name = opts.name; if (Object.hasOwn(opts.service, "image")) { - throw refused("a service may not declare both image and build."); + throw refused(name, "a service may not declare both image and build."); } - const { context, dockerfile, target, args } = buildFields(build); + const { context, dockerfile, target, args } = buildFields(name, build); if (context.includes("://") || context.startsWith("git@")) { - throw refused("remote build contexts are not supported."); + throw refused(name, "remote build contexts are not supported."); } if (isAbsolute(dockerfile)) { - throw refused("the Dockerfile must be relative to the build context."); + throw refused( + name, + "the Dockerfile must be relative to the build context." + ); } const absolute = resolve(dirname(opts.composeFile), context); const root = resolve(opts.projectRoot); const inside = relative(root, absolute); if (inside === ".." || inside.startsWith(`..${sep}`) || isAbsolute(inside)) { - throw refused("the build context must be inside the project."); + throw refused(name, "the build context must be inside the project."); } return { context: inside === "" ? "." : inside, dockerfile, target, args }; } @@ -209,14 +243,14 @@ export function admitNativeProjectBuilds(opts: { }): ReadonlyMap { const builds = new Map(); for (const [name, service] of Object.entries(opts.services)) { - const request = admitNativeProjectBuild({ ...opts, service }); + const request = admitNativeProjectBuild({ ...opts, name, service }); if (request === undefined) { continue; } if (!opts.enabled) { throw new HackCliError({ code: "E_NATIVE_PROJECT_UNSUPPORTED", - message: NATIVE_BUILDS_DISABLED_REFUSAL, + message: nativeBuildsDisabledRefusal(name), }); } builds.set(name, request); @@ -461,7 +495,7 @@ export function refuseManagedSecretBuildArguments(opts: { if (secrets.has(value)) { throw new HackCliError({ code: "E_NATIVE_PROJECT_UNSUPPORTED", - message: `Native foreground up cannot build ${service}: build arg ${name} equals a hack env secret value. Build args are not secrets (images record them); pass secrets with BuildKit build secrets instead, which native builds do not support yet. Values were omitted; no hook, VM or build was started.`, + message: `Native foreground up cannot build service ${service}: build arg ${name} equals a hack env secret value. Build args are not secrets (images record them); pass secrets with BuildKit build secrets instead, which native builds do not support yet. Values were omitted; no hook, VM or build was started.`, }); } } @@ -591,6 +625,8 @@ async function runPendingBuilds(opts: { cwd: opts.projectRoot, requestFile, signal: opts.signal, + }).catch((error: unknown) => { + throw nameBuildFailure(services, error); }); // The request carries argument values; remove it as soon as it is consumed. await rm(requestFile, { force: true }); @@ -606,6 +642,58 @@ async function runPendingBuilds(opts: { } } +/** + * Runtime refusal codes from the native build source policy: what the runtime + * accepts as build input and as the built image, each with what to change. The + * runtime checks them before or right after the build; they are not failures of + * the build itself. + */ +const BUILD_SOURCE_POLICY: Readonly> = { + native_build_source: + "Keep the build context and Dockerfile inside the project as regular files reached without symlinks.", + native_build_dockerfile: "Change the Dockerfile line it names.", + native_build_base_image: + "Use public Docker Hub images in the Dockerfile's FROM and COPY --from.", + native_build_argument: + "Change the build args in the service's Compose build.", + native_build_reserved_label: + "Remove reserved ownership labels from the Dockerfile and its base images.", + native_build_image_volumes: + "Remove VOLUME from the Dockerfile and its base images; declare volumes in Compose instead.", +}; + +/** + * Name the services of a failed `runtime build-image`. A refusal from the build + * source policy says so and what to change; any other runtime failure keeps its + * message. The native code is kept for callers that classify failures. + * Cancellation and closed client errors carry no service-specific cause and pass + * through unchanged. + */ +function nameBuildFailure( + services: readonly string[], + error: unknown +): unknown { + if (!(error instanceof NativeRuntimeRequestError)) { + return error; + } + const which = serviceList(services); + const code = error.nativeCode; + const change = + code !== undefined && Object.hasOwn(BUILD_SOURCE_POLICY, code) + ? BUILD_SOURCE_POLICY[code] + : undefined; + const message = + change === undefined + ? `Native foreground up could not build ${which}: ${error.message}` + : `Native foreground up cannot build ${which}: the native build source policy refused it. ${error.message} ${change} No image was adopted.`; + return new NativeRuntimeRequestError({ + message, + nativeCode: error.nativeCode, + nativeCauseCode: error.nativeCauseCode, + nativeStopFailures: error.nativeStopFailures, + }); +} + function pin(spec: Record, image: string): void { if (!IMAGE_ID.test(image)) { throw new ClosedCliError( diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 02778adc6..24553e467 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -7,6 +7,7 @@ import { checkLegacyProjectAgentArtifacts, checkLegacyUserAgentArtifacts, } from "../agents/legacy-artifacts.ts"; +import { nativeBuildsEnabled } from "../backends/native-project-builds.ts"; import { runNativeProjectCleanupCommand } from "../backends/native-project-cleanup-command.ts"; import { parseNativeRunMappingRecoveryOptions, @@ -2945,10 +2946,14 @@ async function checkComposeBackendFeatures(opts: { message: `Skipped (no ${HACK_PROJECT_DIR_PRIMARY}/ found)`, }; } + // The same build classification native up applies under HACK_NATIVE_BUILDS. + const buildsEnabled = nativeBuildsEnabled(); return composeBackendCheckResult({ backend: opts.backend, + buildsEnabled, features: await readComposeBackendFeatures(ctx.composeFile, { adaptationFile: opts.adaptationFile, + buildsEnabled, }), }); } diff --git a/src/commands/project.ts b/src/commands/project.ts index ff71f6a36..18894ba4b 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -42,6 +42,7 @@ import { import { buildOnboardingPrompt } from "../agents/onboarding-prompt.ts"; import { composeLogBackend, lokiLogBackend } from "../backends/log-backend.ts"; import { parseNativePreparedBase } from "../backends/native-prepared-base.ts"; +import { nativeBuildsEnabled } from "../backends/native-project-builds.ts"; import { nativeDownEnvironment, nativeProjectDown, @@ -5728,10 +5729,13 @@ async function handleNativeUp({ // stays available for any graph a project already has. // The explicit adaptation applies first: a network it isolates is not // external here, exactly as native start normalizes it. + // With native builds enabled, build fields native builds implement pass + // here and native build admission reviews them, secret-valued args included. await refuseNativeComposeBackendFeatures({ composeFile: project.composeFile, adaptationFile: process.env.HACK_NATIVE_ADAPTATION, restart, + buildsEnabled: nativeBuildsEnabled(), }); const branch = await resolveEffectiveBranchForCommand({ project, diff --git a/src/lib/doctor-compose-backend.ts b/src/lib/doctor-compose-backend.ts index da78f14b8..86765332b 100644 --- a/src/lib/doctor-compose-backend.ts +++ b/src/lib/doctor-compose-backend.ts @@ -1,6 +1,7 @@ import { COMPOSE_BACKEND_SELECTION, type ComposeBackendFeature, + type ComposeBackendFeatureOptions, describeComposeBackendFeatures, } from "./native-compose-backend-features.ts"; @@ -21,10 +22,12 @@ export type ComposeBackendReport = { * refuses the project; on the Compose backend it is informational, because * the project already runs where those features are supported. */ -export function composeBackendCheckResult(opts: { - readonly backend: "native" | "compose"; - readonly features: readonly ComposeBackendFeature[]; -}): { +export function composeBackendCheckResult( + opts: { + readonly backend: "native" | "compose"; + readonly features: readonly ComposeBackendFeature[]; + } & ComposeBackendFeatureOptions +): { readonly name: string; readonly status: "ok" | "warn"; readonly message: string; @@ -40,7 +43,7 @@ export function composeBackendCheckResult(opts: { composeBackend, }; } - const uses = `This project uses ${describeComposeBackendFeatures(opts.features)}`; + const uses = `This project uses ${describeComposeBackendFeatures(opts.features, opts)}`; if (opts.backend === "native") { return { name: COMPOSE_BACKEND_CHECK, diff --git a/src/lib/native-compose-backend-features.ts b/src/lib/native-compose-backend-features.ts index 2afef1ed8..2cf759832 100644 --- a/src/lib/native-compose-backend-features.ts +++ b/src/lib/native-compose-backend-features.ts @@ -1,5 +1,6 @@ import { constants } from "node:fs"; import { open } from "node:fs/promises"; +import { NATIVE_BUILD_FIELDS } from "../backends/native-project-builds.ts"; import { readNativeSelection } from "../backends/native-project-selection.ts"; import { HackCliError } from "./cli-result.ts"; import { isRecord } from "./guards.ts"; @@ -73,6 +74,25 @@ const FEATURE_PHRASES: Readonly> = { foreign_platform: "a service platform other than linux/arm64", }; +/** "a, b, c and d" for the build fields native builds implement. */ +function nativeBuildFieldList(): string { + return `${NATIVE_BUILD_FIELDS.slice(0, -1).join(", ")} and ${NATIVE_BUILD_FIELDS.at(-1)}`; +} + +/** The `unsupported_build` phrase while `HACK_NATIVE_BUILDS=1` is set. */ +const NATIVE_BUILDS_PHRASE = `a Compose build field other than ${nativeBuildFieldList()}, or image together with build`; + +/** Options shared by the classifier, its readers and its messages. */ +export type ComposeBackendFeatureOptions = { + /** + * Whether native builds are enabled (`HACK_NATIVE_BUILDS=1`). Then a build + * is a Compose-backend feature only when it declares a field native builds + * do not implement, or `image` together with `build`; native build + * admission reviews the rest, including `args` and the managed-secret check. + */ + readonly buildsEnabled?: boolean; +}; + function labelKeys(labels: unknown): readonly string[] { if (isRecord(labels)) { return Object.keys(labels); @@ -109,19 +129,36 @@ function isExternalNetwork(network: unknown): boolean { } /** - * A build is inside the supported path only when the import preview's basic - * build mapper accepts it and the service names no image, exactly as - * `hack config import --dry-run` reports `invalid_or_unsupported_build` and - * `image_build_exclusive`. + * Without native builds, a build is inside the supported path only when the + * import preview's basic build mapper accepts it and the service names no + * image, exactly as `hack config import --dry-run` reports + * `invalid_or_unsupported_build` and `image_build_exclusive`. + * + * With native builds, the gate admits exactly the fields native builds + * implement ({@link NATIVE_BUILD_FIELDS}). Values in those fields, such as + * paths, interpolation and build args, are left to native build admission, + * which refuses with the service name and never suggests the Compose backend: + * a build arg that carries a secret would be recorded in the image history + * there too. */ -function isUnsupportedBuild(service: Record): boolean { +function isUnsupportedBuild( + service: Record, + buildsEnabled: boolean +): boolean { if (!Object.hasOwn(service, "build")) { return false; } - return ( - Object.hasOwn(service, "image") || - mapLegacyComposeBuild(service.build) === undefined - ); + if (Object.hasOwn(service, "image")) { + return true; + } + const build = service.build; + if (buildsEnabled) { + const fields: ReadonlySet = new Set(NATIVE_BUILD_FIELDS); + return ( + isRecord(build) && Object.keys(build).some((key) => !fields.has(key)) + ); + } + return mapLegacyComposeBuild(build) === undefined; } /** @@ -174,7 +211,7 @@ export function findComposeBackendFeatures( * {@link readAdaptedIsolatedNetworks}. */ readonly isolatedNetworks?: ReadonlySet; - } + } & ComposeBackendFeatureOptions ): ComposeBackendFeature[] { if (!isRecord(compose)) { return []; @@ -205,7 +242,8 @@ export function findComposeBackendFeatures( ) { found.add("reserved_label"); } - if (services.some(isUnsupportedBuild)) { + const buildsEnabled = opts?.buildsEnabled === true; + if (services.some((service) => isUnsupportedBuild(service, buildsEnabled))) { found.add("unsupported_build"); } if (services.some(isForeignPlatform)) { @@ -305,10 +343,11 @@ export async function readAdaptedIsolatedNetworks( */ export async function readComposeBackendFeatures( composeFile: string, - opts?: { readonly adaptationFile?: string } + opts?: { readonly adaptationFile?: string } & ComposeBackendFeatureOptions ): Promise { const compose = await readComposeDocument(composeFile); - const features = findComposeBackendFeatures(compose); + const buildsEnabled = opts?.buildsEnabled; + const features = findComposeBackendFeatures(compose, { buildsEnabled }); if ( opts?.adaptationFile === undefined || !features.includes("external_network") @@ -317,14 +356,20 @@ export async function readComposeBackendFeatures( } return findComposeBackendFeatures(compose, { isolatedNetworks: await readAdaptedIsolatedNetworks(opts.adaptationFile), + buildsEnabled, }); } /** Value-free phrase list, such as "a and b" or "a, b and c". */ export function describeComposeBackendFeatures( - features: readonly ComposeBackendFeature[] + features: readonly ComposeBackendFeature[], + opts?: ComposeBackendFeatureOptions ): string { - const phrases = features.map((feature) => FEATURE_PHRASES[feature]); + const phrases = features.map((feature) => + feature === "unsupported_build" && opts?.buildsEnabled === true + ? NATIVE_BUILDS_PHRASE + : FEATURE_PHRASES[feature] + ); if (phrases.length <= 1) { return phrases[0] ?? ""; } @@ -340,14 +385,16 @@ export const COMPOSE_BACKEND_SELECTION = * `E_NATIVE_PROJECT_UNSUPPORTED`, one line, and a `detail` that lists only the * closed feature names. */ -export function composeBackendRefusal(opts: { - readonly features: readonly ComposeBackendFeature[]; - readonly restart: boolean; -}): HackCliError { +export function composeBackendRefusal( + opts: { + readonly features: readonly ComposeBackendFeature[]; + readonly restart: boolean; + } & ComposeBackendFeatureOptions +): HackCliError { const command = opts.restart ? "hack restart" : "hack up"; return new HackCliError({ code: "E_NATIVE_PROJECT_UNSUPPORTED", - message: `Native \`${command}\` does not support ${describeComposeBackendFeatures(opts.features)}; this project runs on the Compose backend. ${COMPOSE_BACKEND_SELECTION} and run the command again. Nothing was started or stopped.`, + message: `Native \`${command}\` does not support ${describeComposeBackendFeatures(opts.features, opts)}; this project runs on the Compose backend. ${COMPOSE_BACKEND_SELECTION} and run the command again. Nothing was started or stopped.`, detail: { backend: "compose", features: [...opts.features] }, }); } @@ -357,16 +404,16 @@ export function composeBackendRefusal(opts: { * needs the Compose backend. Call it after project selection and before any * runtime request, storage preparation, lifecycle hook or stop. */ -export async function refuseNativeComposeBackendFeatures(opts: { - readonly composeFile: string; - /** The `HACK_NATIVE_ADAPTATION` path native start applies, if any. */ - readonly adaptationFile?: string; - readonly restart: boolean; -}): Promise { - const features = await readComposeBackendFeatures(opts.composeFile, { - adaptationFile: opts.adaptationFile, - }); +export async function refuseNativeComposeBackendFeatures( + opts: { + readonly composeFile: string; + /** The `HACK_NATIVE_ADAPTATION` path native start applies, if any. */ + readonly adaptationFile?: string; + readonly restart: boolean; + } & ComposeBackendFeatureOptions +): Promise { + const features = await readComposeBackendFeatures(opts.composeFile, opts); if (features.length > 0) { - throw composeBackendRefusal({ features, restart: opts.restart }); + throw composeBackendRefusal({ ...opts, features }); } } diff --git a/tests/native-compose-backend-features-cli.test.ts b/tests/native-compose-backend-features-cli.test.ts index 0ef0e5a26..bf4e063c8 100644 --- a/tests/native-compose-backend-features-cli.test.ts +++ b/tests/native-compose-backend-features-cli.test.ts @@ -10,8 +10,10 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, relative, resolve } from "node:path"; -import { NATIVE_BUILDS_DISABLED_REFUSAL } from "../src/backends/native-project-builds.ts"; +import { nativeBuildsDisabledRefusal } from "../src/backends/native-project-builds.ts"; import { NATIVE_CONFIGURATION_REFUSAL } from "../src/backends/native-project-start.ts"; +import { PROJECT_ENV_KEY_FILENAME } from "../src/constants.ts"; +import { setProjectEnvValue } from "../src/lib/project-env-config.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; /** @@ -332,7 +334,7 @@ test( NATIVE_CONFIGURATION_REFUSAL, ], // A basic build stays refused unless native builds are explicitly enabled. - [{ build: ".." }, NATIVE_BUILDS_DISABLED_REFUSAL], + [{ build: ".." }, nativeBuildsDisabledRefusal("web")], ] as const) { const f = await fixture({ services: { web: service } }); const result = await cli(f, ["up"]); @@ -345,6 +347,86 @@ test( SPAWN_TIMEOUT_MS ); +/** Native builds enabled, with the usual shared-source grant. */ +const BUILDS_ENV = { HACK_NATIVE_SHARED_SOURCE: "1", HACK_NATIVE_BUILDS: "1" }; + +test( + "with native builds enabled, build args pass the check and reach native start", + async () => { + // HACK-1925: args are a native builds field, so the Compose-backend check + // passes and native start goes on to its own source preflight and runtime. + const f = await fixture({ + services: { + web: { + build: { context: "../app", args: { GREETING: "synthetic" } }, + }, + }, + }); + const result = await cli(f, ["up"], BUILDS_ENV); + expect(result.output).not.toContain("E_NATIVE_PROJECT_UNSUPPORTED"); + expect(result.output).not.toContain("HACK_RUNTIME_BACKEND=compose"); + expect(result.output).toMatch(RUNTIME_REQUEST_LINE); + const calls = await f.calls(); + expect(calls).toEqual( + ["runtime check-project-share", "runtime up"].slice(0, calls.length) + ); + expect(calls.length).toBeGreaterThan(0); + }, + SPAWN_TIMEOUT_MS +); + +test( + "with native builds enabled, a build arg equal to a managed secret refuses by name only", + async () => { + const secret = "synthetic-dummy-managed-value-1925"; + const f = await fixture({ + services: { + web: { build: { context: "../app", args: { GREETING: secret } } }, + }, + }); + await writeFile( + join(f.project, PROJECT_ENV_KEY_FILENAME), + "synthetic-project-key" + ); + await setProjectEnvValue({ + projectRoot: f.project, + projectDir: join(f.project, ".hack"), + envName: null, + scope: "global", + key: "FIXTURE_TOKEN", + value: secret, + secret: true, + }); + const result = await cli(f, ["up"], BUILDS_ENV); + expectOneClosedLine( + result, + "E_NATIVE_PROJECT_UNSUPPORTED: Native foreground up cannot build service web: build arg GREETING equals a hack env secret value. Build args are not secrets (images record them); pass secrets with BuildKit build secrets instead, which native builds do not support yet. Values were omitted; no hook, VM or build was started." + ); + expect(result.output).not.toContain(secret); + // The Compose backend would record the value in the image history too. + expect(result.output).not.toContain("HACK_RUNTIME_BACKEND=compose"); + expect(await f.calls()).toEqual(["runtime check-project-share"]); + }, + SPAWN_TIMEOUT_MS +); + +test( + "with native builds enabled, a build field native builds lack still selects the Compose backend", + async () => { + await expectComposeBackendRefusal({ + compose: { + services: { + web: { build: { context: "../app", ssh: ["default"] } }, + }, + }, + env: BUILDS_ENV, + phrase: + "a Compose build field other than context, dockerfile, target and args, or image together with build", + }); + }, + SPAWN_TIMEOUT_MS +); + test( "control: a reviewed isolated network replacement and arm64 platform pass the check", async () => { diff --git a/tests/native-compose-backend-features.test.ts b/tests/native-compose-backend-features.test.ts index 34c766d22..b585f6910 100644 --- a/tests/native-compose-backend-features.test.ts +++ b/tests/native-compose-backend-features.test.ts @@ -166,6 +166,64 @@ test("builds follow the import preview's basic build path", () => { } }); +test("with native builds enabled, exactly the fields native builds implement pass", () => { + const enabled = { buildsEnabled: true } as const; + // HACK-1925: args and the other implemented fields reach native build admission, + // which reviews values (paths, interpolation, managed secrets) itself. + for (const build of [ + "..", + "/absolute/context", + { context: "../app", args: { GREETING: "synthetic" } }, + { context: "..", args: ["A=1"], dockerfile: "Dockerfile", target: "dev" }, + { context: "../../outside" }, + { args: { A: "${HOST_VALUE}" } }, + {}, + ]) { + expect(findComposeBackendFeatures(services({ build }), enabled)).toEqual( + [] + ); + } + for (const service of [ + { build: { context: "..", platforms: ["linux/amd64"] } }, + { build: { context: "..", ssh: ["default"] } }, + { build: { context: "..", secrets: ["token"] } }, + { build: { context: "..", args: { A: "1" }, cache_from: ["x"] } }, + { build: "..", image: IMAGE }, + ]) { + expect(findComposeBackendFeatures(services(service), enabled)).toEqual([ + "unsupported_build", + ]); + } + // Without native builds, today's basic-form classification is unchanged. + expect( + findComposeBackendFeatures( + services({ build: { context: "..", args: { A: "1" } } }) + ) + ).toEqual(["unsupported_build"]); +}); + +test("the build refusal phrase follows whether native builds are enabled", () => { + const features = ["unsupported_build"] as const; + expect(composeBackendRefusal({ features, restart: false }).message).toContain( + "does not support a Compose build outside the basic context, dockerfile and target form;" + ); + const enabled = composeBackendRefusal({ + features, + restart: false, + buildsEnabled: true, + }).message; + expect(enabled).toContain( + "does not support a Compose build field other than context, dockerfile, target and args, or image together with build;" + ); + expect( + composeBackendCheckResult({ + backend: "native", + features, + buildsEnabled: true, + }).message + ).toContain("a Compose build field other than context, dockerfile"); +}); + test("only the native linux/arm64 platform passes", () => { for (const platform of ["linux/arm64", "linux/arm64/v8"]) { expect( diff --git a/tests/native-project-builds.test.ts b/tests/native-project-builds.test.ts index c281fb319..9dc3c8f35 100644 --- a/tests/native-project-builds.test.ts +++ b/tests/native-project-builds.test.ts @@ -15,8 +15,8 @@ import { admitNativeProjectBuilds, buildNativeProjectImages, invokeNativeBuild, - NATIVE_BUILDS_DISABLED_REFUSAL, type NativeBuildRequest, + nativeBuildsDisabledRefusal, nativeBuildsEnabled, nativeServiceActive, refuseManagedSecretBuildArguments, @@ -45,7 +45,12 @@ function admit( service: Record, composeFile = "/p/.hack/compose.yaml" ) { - return admitNativeProjectBuild({ service, projectRoot: "/p", composeFile }); + return admitNativeProjectBuild({ + name: "app", + service, + projectRoot: "/p", + composeFile, + }); } test("admitted build forms follow Compose paths and literal arguments", () => { @@ -132,8 +137,14 @@ test.each([ })(); expect(error).toBeInstanceOf(HackCliError); expect(error).toMatchObject({ code: "E_NATIVE_PROJECT_UNSUPPORTED" }); - expect(String((error as Error).message)).toContain(text); - expect(String((error as Error).message)).not.toContain("HOST_VALUE"); + const message = String((error as Error).message); + expect(message).toContain(text); + // HACK-1927: every admission refusal names the service it is about. + expect(message).toStartWith( + "Native foreground up cannot build service app: " + ); + expect(message).not.toContain("HOST_VALUE"); + expect(message).not.toContain("HACK_RUNTIME_BACKEND"); }); test("builds refuse unless explicitly enabled, and projects without builds are unaffected", () => { @@ -145,7 +156,10 @@ test("builds refuse unless explicitly enabled, and projects without builds are u composeFile: "/p/compose.yaml", enabled: false, }) - ).toThrow(NATIVE_BUILDS_DISABLED_REFUSAL); + ).toThrow(nativeBuildsDisabledRefusal("web")); + expect(nativeBuildsDisabledRefusal("web")).toContain( + "cannot build service web: " + ); expect( admitNativeProjectBuilds({ services: { db: { image: "postgres:17" } }, @@ -367,10 +381,14 @@ test("a build arg equal to a managed secret value refuses without showing it", ( expect(refusal).toBeInstanceOf(HackCliError); const error = refusal as HackCliError; expect(error.code).toBe("E_NATIVE_PROJECT_UNSUPPORTED"); + // HACK-1925: the refusal names the service and the argument, never the value, + // and does not point at the Compose backend, which would bake it into history. expect(error.message).toContain( - "build arg TOKEN equals a hack env secret value" + "cannot build service web: build arg TOKEN equals a hack env secret value" ); expect(error.message).not.toContain(secret); + expect(error.message).not.toContain("HACK_RUNTIME_BACKEND"); + expect(error.message).not.toContain("Compose backend"); // Values that are not managed secrets pass; so does any build with no secrets. refuseManagedSecretBuildArguments({ builds: new Map([["web", request({ MODE: "production" })]]), @@ -460,3 +478,57 @@ test("cancellation sends SIGTERM so the runtime can stop its build client", asyn expect(String(error)).toContain("canceled during an image build"); expect(await readFile(`${fake.binary}.signal`, "utf8")).toBe("terminated"); }); + +test("runtime build refusals name the services and the build source policy", async () => { + const project = await root(); + const policy = + "Dockerfile line 1: build images must be public Docker Hub references."; + const fail = (nativeCode: string, message: string) => async () => { + throw new NativeRuntimeRequestError({ message, nativeCode }); + }; + const run = async (invoke: () => Promise) => + await buildNativeProjectImages({ + runtime, + projectRoot: project, + branch: null, + profiles: [], + // Identical declarations build once, so one refusal covers both services. + builds: new Map([ + ["web", request()], + ["worker", request()], + ]), + specs: { web: { build: "app" }, worker: { build: "app" } }, + retainedImages: new Map(), + restore: false, + rebuild: false, + signal: new AbortController().signal, + invoke, + }).catch((caught: unknown) => caught); + + const refused = await run(fail("native_build_base_image", policy)); + expect(refused).toBeInstanceOf(NativeRuntimeRequestError); + expect(refused).toMatchObject({ nativeCode: "native_build_base_image" }); + const message = (refused as Error).message; + expect(message).toStartWith( + "Native foreground up cannot build services web and worker: the native build source policy refused it. " + ); + expect(message).toContain(policy); + expect(message).toContain("Use public Docker Hub images"); + expect(message).not.toContain(VALUE); + + // A failure that is not a policy refusal names the services and keeps its message. + const failed = await run( + fail("native_build_failed", "The image build failed.") + ); + expect(failed).toMatchObject({ nativeCode: "native_build_failed" }); + expect((failed as Error).message).toBe( + "Native foreground up could not build services web and worker: The image build failed." + ); + expect((failed as Error).message).not.toContain("source policy"); + + // An inherited property name is not a policy code. + const odd = await run(fail("constructor", "Unexpected failure.")); + expect((odd as Error).message).toBe( + "Native foreground up could not build services web and worker: Unexpected failure." + ); +});