diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1120940..5ff1dfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,27 +28,15 @@ jobs: - name: Reject tracked local-only files run: scripts/ci/guard-tracked-files.sh - - name: Run guard regression tests - run: scripts/ci/tests/guard-tracked-files.test.sh - - name: Require a digest-pinned Dockerfile frontend run: scripts/ci/guard-dockerfile-frontend.sh - - name: Run Dockerfile frontend guard regression tests - run: scripts/ci/tests/guard-dockerfile-frontend.test.sh - - name: Check the container scan exceptions against their policy run: scripts/ci/guard-trivyignore.sh - - name: Run container scan exception guard regression tests - run: scripts/ci/tests/guard-trivyignore.test.sh - - name: Keep the edge configuration free of ACME server, PKI, issuers and mutable admin run: scripts/ci/guard-edge-config.sh - - name: Run edge configuration guard regression tests - run: scripts/ci/tests/guard-edge-config.test.sh - - name: ShellCheck run: >- docker run --rm --network none -v "$PWD:/mnt:ro" -w /mnt @@ -57,6 +45,7 @@ jobs: scripts/ci/guard-dockerfile-frontend.sh scripts/ci/tests/guard-dockerfile-frontend.test.sh scripts/ci/guard-trivyignore.sh scripts/ci/tests/guard-trivyignore.test.sh scripts/ci/guard-edge-config.sh scripts/ci/tests/guard-edge-config.test.sh + scripts/ci/no-docker.sh scripts/ci/tests/no-docker.test.sh - name: actionlint run: >- @@ -64,6 +53,26 @@ jobs: rhysd/actionlint:1.7.12@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667 -color + # Code under test runs only in the steps from here to the end of the + # job, each isolated by scripts/ci/no-docker.sh; no step that is not + # isolated may follow them (checked by scripts/ci/tests/ci-isolation-order.py). + # Each starts through scripts/ci/fdnorm.py: runner steps can inherit + # descriptors above 2, which the isolation launcher refuses. + - name: Run guard regression tests + run: python3 -I scripts/ci/fdnorm.py -- scripts/ci/tests/guard-tracked-files.test.sh + + - name: Run Dockerfile frontend guard regression tests + run: python3 -I scripts/ci/fdnorm.py -- scripts/ci/tests/guard-dockerfile-frontend.test.sh + + - name: Run container scan exception guard regression tests + run: python3 -I scripts/ci/fdnorm.py -- scripts/ci/tests/guard-trivyignore.test.sh + + - name: Run edge configuration guard regression tests + run: python3 -I scripts/ci/fdnorm.py -- scripts/ci/tests/guard-edge-config.test.sh + + - name: Run test isolation regression tests + run: python3 -I scripts/ci/fdnorm.py -- scripts/ci/tests/no-docker.test.sh + # The application checks run in this required job on purpose: a separate # job that is not a required check could fail (or be skipped) without # blocking a merge. @@ -105,9 +114,6 @@ jobs: - name: make help and make doctor run: make help && make doctor - - name: Allocator unit tests - run: tests/devenv/unit.sh - - name: Allocator integration tests (real Docker) run: tests/devenv/integration.sh @@ -127,3 +133,9 @@ jobs: # docs/security/container-vulnerabilities.md). - name: Container vulnerability scan (Trivy) run: tests/app/scan.sh + + # Code under test runs only from here to the end of the job, isolated by + # scripts/ci/no-docker.sh (order checked by scripts/ci/tests/ci-isolation-order.py). + # It starts through scripts/ci/fdnorm.py, as in the Repository guard job. + - name: Allocator unit tests + run: python3 -I scripts/ci/fdnorm.py -- tests/devenv/unit.sh diff --git a/backend/composer.lock b/backend/composer.lock index 189b382..5884706 100644 --- a/backend/composer.lock +++ b/backend/composer.lock @@ -1741,16 +1741,16 @@ }, { "name": "symfony/config", - "version": "v7.4.17", + "version": "v7.4.23", "source": { "type": "git", "url": "https://github.com/symfony/config.git", - "reference": "696e12da8eea497a1a3808d714b43ff67a7df43e" + "reference": "8ef666abf731e85e59cf478929dda6c7cb2fa9c4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/config/zipball/696e12da8eea497a1a3808d714b43ff67a7df43e", - "reference": "696e12da8eea497a1a3808d714b43ff67a7df43e", + "url": "https://api.github.com/repos/symfony/config/zipball/8ef666abf731e85e59cf478929dda6c7cb2fa9c4", + "reference": "8ef666abf731e85e59cf478929dda6c7cb2fa9c4", "shasum": "" }, "require": { @@ -1796,7 +1796,7 @@ "description": "Helps you find, load, combine, autofill and validate configuration values of any kind", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/config/tree/v7.4.17" + "source": "https://github.com/symfony/config/tree/v7.4.23" }, "funding": [ { @@ -1816,7 +1816,7 @@ "type": "tidelift" } ], - "time": "2026-08-20T09:55:18+00:00" + "time": "2026-11-04T08:27:22+00:00" }, { "name": "symfony/console", @@ -2511,16 +2511,16 @@ }, { "name": "symfony/filesystem", - "version": "v7.4.21", + "version": "v7.4.23", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "fd0e3462be6d32a0d130e8423e4940a7688162d8" + "reference": "53d609da66578dad5a39fe290e8f924ec8f05e93" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/fd0e3462be6d32a0d130e8423e4940a7688162d8", - "reference": "fd0e3462be6d32a0d130e8423e4940a7688162d8", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/53d609da66578dad5a39fe290e8f924ec8f05e93", + "reference": "53d609da66578dad5a39fe290e8f924ec8f05e93", "shasum": "" }, "require": { @@ -2557,7 +2557,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v7.4.21" + "source": "https://github.com/symfony/filesystem/tree/v7.4.23" }, "funding": [ { @@ -2577,7 +2577,7 @@ "type": "tidelift" } ], - "time": "2026-10-06T20:05:19+00:00" + "time": "2026-11-04T08:27:22+00:00" }, { "name": "symfony/finder", @@ -2722,16 +2722,16 @@ }, { "name": "symfony/framework-bundle", - "version": "v7.4.21", + "version": "v7.4.23", "source": { "type": "git", "url": "https://github.com/symfony/framework-bundle.git", - "reference": "62b38e9cb02c944e01030898b3d2b4f31722e47f" + "reference": "6a749f2c37cb9def53793633ec72958aaad4b359" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/framework-bundle/zipball/62b38e9cb02c944e01030898b3d2b4f31722e47f", - "reference": "62b38e9cb02c944e01030898b3d2b4f31722e47f", + "url": "https://api.github.com/repos/symfony/framework-bundle/zipball/6a749f2c37cb9def53793633ec72958aaad4b359", + "reference": "6a749f2c37cb9def53793633ec72958aaad4b359", "shasum": "" }, "require": { @@ -2746,7 +2746,7 @@ "symfony/event-dispatcher": "^6.4|^7.0|^8.0", "symfony/filesystem": "^7.1|^8.0", "symfony/finder": "^6.4|^7.0|^8.0", - "symfony/http-foundation": "^7.4|^8.0", + "symfony/http-foundation": "^7.4.23|^8.1.11", "symfony/http-kernel": "^7.4|^8.0", "symfony/polyfill-mbstring": "~1.0", "symfony/polyfill-php85": "^1.32", @@ -2855,7 +2855,7 @@ "description": "Provides a tight integration between Symfony components and the Symfony full-stack framework", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/framework-bundle/tree/v7.4.21" + "source": "https://github.com/symfony/framework-bundle/tree/v7.4.23" }, "funding": [ { @@ -2875,20 +2875,20 @@ "type": "tidelift" } ], - "time": "2026-10-12T12:34:30+00:00" + "time": "2026-11-04T07:46:24+00:00" }, { "name": "symfony/http-foundation", - "version": "v7.4.21", + "version": "v7.4.23", "source": { "type": "git", "url": "https://github.com/symfony/http-foundation.git", - "reference": "58c89144c7a36dd89c3d01917e9fa3a72d8aaf31" + "reference": "7fb73cffa06e2bcc2cb523fcccb1e8174106b206" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-foundation/zipball/58c89144c7a36dd89c3d01917e9fa3a72d8aaf31", - "reference": "58c89144c7a36dd89c3d01917e9fa3a72d8aaf31", + "url": "https://api.github.com/repos/symfony/http-foundation/zipball/7fb73cffa06e2bcc2cb523fcccb1e8174106b206", + "reference": "7fb73cffa06e2bcc2cb523fcccb1e8174106b206", "shasum": "" }, "require": { @@ -2937,7 +2937,7 @@ "description": "Defines an object-oriented layer for the HTTP specification", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/http-foundation/tree/v7.4.21" + "source": "https://github.com/symfony/http-foundation/tree/v7.4.23" }, "funding": [ { @@ -2957,20 +2957,20 @@ "type": "tidelift" } ], - "time": "2026-10-08T16:56:12+00:00" + "time": "2026-11-04T08:27:22+00:00" }, { "name": "symfony/http-kernel", - "version": "v7.4.21", + "version": "v7.4.23", "source": { "type": "git", "url": "https://github.com/symfony/http-kernel.git", - "reference": "4779fce7dedacb8801b2b61c703ef99159502331" + "reference": "94c57f4c5c45a311d244a0aeadb72b14ee41d25d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-kernel/zipball/4779fce7dedacb8801b2b61c703ef99159502331", - "reference": "4779fce7dedacb8801b2b61c703ef99159502331", + "url": "https://api.github.com/repos/symfony/http-kernel/zipball/94c57f4c5c45a311d244a0aeadb72b14ee41d25d", + "reference": "94c57f4c5c45a311d244a0aeadb72b14ee41d25d", "shasum": "" }, "require": { @@ -3056,7 +3056,7 @@ "description": "Provides a structured process for converting a Request into a Response", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/http-kernel/tree/v7.4.21" + "source": "https://github.com/symfony/http-kernel/tree/v7.4.23" }, "funding": [ { @@ -3076,7 +3076,7 @@ "type": "tidelift" } ], - "time": "2026-10-14T06:44:08+00:00" + "time": "2026-11-04T11:26:18+00:00" }, { "name": "symfony/lock", @@ -4373,16 +4373,16 @@ }, { "name": "symfony/routing", - "version": "v7.4.21", + "version": "v7.4.23", "source": { "type": "git", "url": "https://github.com/symfony/routing.git", - "reference": "7a344cf871532d081875bfdd18f3dff8dca36c5b" + "reference": "0000ade604b7bc798cb1e75edf6f17df48185f73" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/routing/zipball/7a344cf871532d081875bfdd18f3dff8dca36c5b", - "reference": "7a344cf871532d081875bfdd18f3dff8dca36c5b", + "url": "https://api.github.com/repos/symfony/routing/zipball/0000ade604b7bc798cb1e75edf6f17df48185f73", + "reference": "0000ade604b7bc798cb1e75edf6f17df48185f73", "shasum": "" }, "require": { @@ -4434,7 +4434,7 @@ "url" ], "support": { - "source": "https://github.com/symfony/routing/tree/v7.4.21" + "source": "https://github.com/symfony/routing/tree/v7.4.23" }, "funding": [ { @@ -4454,7 +4454,7 @@ "type": "tidelift" } ], - "time": "2026-10-11T18:40:59+00:00" + "time": "2026-11-04T08:27:22+00:00" }, { "name": "symfony/runtime", @@ -4822,16 +4822,16 @@ }, { "name": "symfony/security-http", - "version": "v7.4.21", + "version": "v7.4.23", "source": { "type": "git", "url": "https://github.com/symfony/security-http.git", - "reference": "07fcd6b1428a8f0c84f5cb98da6428abb8b89199" + "reference": "ea0623a02764025268bd928810f012767ea1965a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/security-http/zipball/07fcd6b1428a8f0c84f5cb98da6428abb8b89199", - "reference": "07fcd6b1428a8f0c84f5cb98da6428abb8b89199", + "url": "https://api.github.com/repos/symfony/security-http/zipball/ea0623a02764025268bd928810f012767ea1965a", + "reference": "ea0623a02764025268bd928810f012767ea1965a", "shasum": "" }, "require": { @@ -4891,7 +4891,7 @@ "description": "Symfony Security Component - HTTP Integration", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/security-http/tree/v7.4.21" + "source": "https://github.com/symfony/security-http/tree/v7.4.23" }, "funding": [ { @@ -4911,7 +4911,7 @@ "type": "tidelift" } ], - "time": "2026-10-12T12:34:30+00:00" + "time": "2026-11-04T08:27:22+00:00" }, { "name": "symfony/service-contracts", @@ -5488,16 +5488,16 @@ }, { "name": "symfony/yaml", - "version": "v7.4.21", + "version": "v7.4.23", "source": { "type": "git", "url": "https://github.com/symfony/yaml.git", - "reference": "044907929c7999814378be22da34722c7a86bad4" + "reference": "1e601207ffe627a2e621faa9d661f1d32dcf1cec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/yaml/zipball/044907929c7999814378be22da34722c7a86bad4", - "reference": "044907929c7999814378be22da34722c7a86bad4", + "url": "https://api.github.com/repos/symfony/yaml/zipball/1e601207ffe627a2e621faa9d661f1d32dcf1cec", + "reference": "1e601207ffe627a2e621faa9d661f1d32dcf1cec", "shasum": "" }, "require": { @@ -5540,7 +5540,7 @@ "description": "Loads and dumps YAML files", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/yaml/tree/v7.4.21" + "source": "https://github.com/symfony/yaml/tree/v7.4.23" }, "funding": [ { @@ -5560,7 +5560,7 @@ "type": "tidelift" } ], - "time": "2026-10-12T12:34:30+00:00" + "time": "2026-11-04T08:27:22+00:00" } ], "packages-dev": [ diff --git a/backend/tests/Integration/Security/HeadRequestTest.php b/backend/tests/Integration/Security/HeadRequestTest.php new file mode 100644 index 0000000..230ca90 --- /dev/null +++ b/backend/tests/Integration/Security/HeadRequestTest.php @@ -0,0 +1,49 @@ +request('GET', '/api/v1/me')->status); + + $head = self::$fpm->request('HEAD', '/api/v1/me'); + + self::assertSame(401, $head->status); + self::assertSame('', $head->body); + } + + public function testAHeadRequestToAForeignOrganizationIsAnsweredLikeGet(): void + { + $own = $this->fixtures->organization('own'); + $foreign = $this->fixtures->organization('foreign'); + $user = $this->fixtures->user('head-member'); + $this->fixtures->member($own, $user, 'REP'); + $client = $this->client(); + $client->login($user->email, $user->password); + $cookies = [ApiClient::COOKIE => (string) $client->sessionId()]; + + $ownUri = '/api/v1/organizations/'.$own->toRfc4122(); + $foreignUri = '/api/v1/organizations/'.$foreign->toRfc4122(); + + self::assertSame(200, self::$fpm->request('GET', $ownUri, $cookies)->status); + $head = self::$fpm->request('HEAD', $ownUri, $cookies); + self::assertSame(200, $head->status); + self::assertSame('', $head->body); + + self::assertSame(404, self::$fpm->request('GET', $foreignUri, $cookies)->status); + self::assertSame(404, self::$fpm->request('HEAD', $foreignUri, $cookies)->status); + } +} diff --git a/backend/tests/Unit/Shared/Http/Security/MethodMatcherHeadTest.php b/backend/tests/Unit/Shared/Http/Security/MethodMatcherHeadTest.php new file mode 100644 index 0000000..62a0163 --- /dev/null +++ b/backend/tests/Unit/Shared/Http/Security/MethodMatcherHeadTest.php @@ -0,0 +1,27 @@ +matches(Request::create('/', 'HEAD'))); + } + + public function testAMatcherForPostDoesNotApplyToHead(): void + { + self::assertFalse((new MethodRequestMatcher('POST'))->matches(Request::create('/', 'HEAD'))); + } +} diff --git a/bin/devenv b/bin/devenv index a943548..aae96cd 100755 --- a/bin/devenv +++ b/bin/devenv @@ -68,33 +68,33 @@ show() { } usage() { - cat <<'EOF' -Usage: bin/devenv - - up Start the stack and print its URL (port chosen automatically) - down Stop and remove this checkout's containers (volumes are kept) - restart down + up, preferring the saved port - status Show container state and the saved port (exit 1 if not running) - url Print the verified URL (exit 1 if not running) - ports Show candidate ports and who uses them (read-only) - logs Show recent logs (SERVICE=, TAIL=200, FOLLOW=1) - doctor Check prerequisites (read-only) - upgrade-roles - Move a database volume created before role separation to the - three roles of ADR-018, after a verified backup (explicit only) - rotate-db-password - New password for ROLE=app or ROLE=owner (development only; - recreates the containers that use it). Run again after an - interruption to finish the rotation - rotate-db-password-abort - Undo an unfinished rotation of ROLE=app or ROLE=owner: the - database gets the current password back - -Configuration: FLOWCRM_STACK (app or fixture), FLOWCRM_PORT_DEFAULT (8080), -FLOWCRM_PORT_RANGE (8080-8099), FLOWCRM_HTTP_PORT (explicit port, no -fallback), FLOWCRM_UP_TIMEOUT (600), FLOWCRM_WAIT_TIMEOUT (60), -FLOWCRM_LOCK_WAIT (120). See docs/dev-environment.md. -EOF + # One argument per line: no here-document (bash 3.2 keeps those in /tmp). + printf '%s\n' \ + 'Usage: bin/devenv ' \ + '' \ + ' up Start the stack and print its URL (port chosen automatically)' \ + ' down Stop and remove this checkout'\''s containers (volumes are kept)' \ + ' restart down + up, preferring the saved port' \ + ' status Show container state and the saved port (exit 1 if not running)' \ + ' url Print the verified URL (exit 1 if not running)' \ + ' ports Show candidate ports and who uses them (read-only)' \ + ' logs Show recent logs (SERVICE=, TAIL=200, FOLLOW=1)' \ + ' doctor Check prerequisites (read-only)' \ + ' upgrade-roles' \ + ' Move a database volume created before role separation to the' \ + ' three roles of ADR-018, after a verified backup (explicit only)' \ + ' rotate-db-password' \ + ' New password for ROLE=app or ROLE=owner (development only;' \ + ' recreates the containers that use it). Run again after an' \ + ' interruption to finish the rotation' \ + ' rotate-db-password-abort' \ + ' Undo an unfinished rotation of ROLE=app or ROLE=owner: the' \ + ' database gets the current password back' \ + '' \ + 'Configuration: FLOWCRM_STACK (app or fixture), FLOWCRM_PORT_DEFAULT (8080),' \ + 'FLOWCRM_PORT_RANGE (8080-8099), FLOWCRM_HTTP_PORT (explicit port, no' \ + 'fallback), FLOWCRM_UP_TIMEOUT (600), FLOWCRM_WAIT_TIMEOUT (60),' \ + 'FLOWCRM_LOCK_WAIT (120). See docs/dev-environment.md.' } # --- validation ------------------------------------------------------------- @@ -393,7 +393,7 @@ proc_start() { local stat boot f if [ -r /proc/sys/kernel/random/boot_id ] && stat=$(cat "/proc/$1/stat" 2>/dev/null); then boot=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null) - read -r -a f <<<"${stat##*) }" + read -r -a f < <(printf '%s\n' "${stat##*) }") if [ -n "$boot" ] && [[ ${f[19]-} =~ ^[0-9]+$ ]]; then printf 'linux:%s:%s\n' "$boot" "${f[19]}"; fi return 0 fi @@ -758,9 +758,7 @@ require_owned_project() { # recorded on each container tells them apart. [ "$files" = "$COMPOSE_FILE" ] || die $EX_SAFETY "project $PROJECT has containers of another stack (container $id from $(show "${files##*/}")); this command manages FLOWCRM_STACK=$STACK ($COMPOSE_FILE_NAME). Stop them first with: FLOWCRM_STACK=$(stack_of_file "$files") make down" - done <`. + - It is loaded only from the pinned template (sha256 constant), added and + never replaced. + - It attaches with `aa-exec` after `setpriv` has dropped privileges, + inside the existing network, mount, pid and IPC namespaces. +2. **A functional proof per launch (the fence).** Before the command + starts, the process that will become the command (`no-docker-probe.py + --gate-exec`) must find two sockets of the launch, bound outside, refused + with `EACCES` exactly. + - File permissions are first shown to allow them (`faccessat`), so the + refusal can only come from the profile. + - Its label must be exactly `NAME (enforce)`. + - A socket it binds inside must connect. + - On a kernel, parser or image where the denial does not hold, every + launch refuses (125). +3. **A descriptor boundary.** The command receives exactly: + - `/dev/null` as stdin, opened inside; + - pipes to relays of `no-docker.sh` as stdout and stderr. + + `--root-exec` closes every other descriptor and verifies it. The gate + checks its own descriptor table last, then execs the command in its own + process, after resetting the signal dispositions. A caller's stdout or + stderr that is a socket or terminal is refused. +4. **A profile lifecycle that never unconfines a live process.** Removing + a profile unconfines any task that still carries it. The profile is + removed only after all of these hold: + - **P0:** a root-owned record of the launch's pid namespace exists, + written by its pid 1 before anything untrusted runs (or by `kill-init` + from the verified pid 1); + - **P1:** no task is in that namespace but its zombie init (the kernel + kills every other task before init becomes a zombie, + `zap_pid_ns_processes`); + - **P2:** the profile is unchanged; + - **sweeps:** two complete system-wide sweeps find no carrier. + + Any doubt keeps the profile loaded and the run fails (125, the control + directory kept and named). The launch state is written before each + transition and read back. +5. **Fail closed throughout.** + - The preflight checks that AppArmor is enabled, in enforce mode, with + the `af_unix` class, and that the tools exist. + - Labels must match exactly. + - One guarded cleanup acts only on what was recorded. + - Exit precedence: a cleanup failure gives 125; the negative control + gives 125; a signal gives 128 plus the signal; a refusal gives 125. +6. **No global AppArmor change.** No reload, replacement, mode change or + configuration write. Static rules (T-39) check this on every run. + +## Alternatives considered + +- **Keep the enumeration and accept its residual, or always run the walk.** + The enumeration is not provably complete, and the walk costs 140 to 150 + seconds per launch. +- **A seccomp filter denying `AF_UNIX`.** It breaks the code under test's + own sockets. +- **Landlock.** Not available for this on the runners' kernel. +- **One profile per job, or never removing profiles.** Leftovers would mix + launches, and removal could not be tied to one launch's namespace. +- **Attaching before `setpriv` (order B).** It would need capability rules + in the profile. + +## Kernel and runner prerequisites + +- AppArmor enabled and in enforce mode, with `features/network/af_unix` + (Ubuntu's v7/v8 class, which the noble parser targets). +- `kernel.apparmor_restrict_unprivileged_unconfined=0`; with 1 the label is + stacked and every launch refuses. +- Passwordless `sudo`, `aa-exec`, `apparmor_parser`, `setpriv` and + `unshare` (util-linux). sudo's default `closefrom` and `env_reset` are + recorded, not relied on. + +## Consequences + +**Positive:** +- The kernel refuses every host-bound pathname socket, including late, + aliased, unlistable and datagram ones; the enumeration's residuals are + gone. +- The guard job's estimated time drops to about 315 to 378 seconds (an + estimate from component bounds, for the 45 launches of the guard suites + and `no-docker.test.sh`; the 600-second timeout is unchanged, and L-16 + fails above a projected 540 seconds). + +**Accepted residual risks** (docs/dev-environment.md): +- **RR-01:** a runner cancellation leaves confined processes running and the + profile loaded until the VM is discarded. +- **RR-02 and RR-03:** kernel or image drift, or the sysctl, make every + launch refuse. +- **RR-04:** io_uring, rename and hard-link paths rest on reading the + kernel source. +- **RR-05:** a path that root makes into the namespace. +- **RR-06:** a late chain after a signal in the start window (its gate + refuses, and the profile stays loaded). +- **RR-07:** a trusted actor attaching after the last sweep. +- **RR-08:** audit records may be lost. +- **RR-09:** the launch chain must stay as designed. +- **RR-10:** annotation ordering between stdout and stderr. +- **RR-11:** untrapped signals. +- **RR-12:** a reused namespace inode (a false 125 only). +- **RR-13:** no namespace record after an internal fault (the profile stays + loaded, with no carrier). + +## Validation plan + +1. Local: the pure and static suites (T-36, T-38, T-39), T-19, ShellCheck + and the macOS cases. These prove logic and structure, not enforcement. +2. A separately authorized run on a GitHub-hosted runner of the four guard + suites, `no-docker.test.sh` (L-01 to L-18, T-16 to T-18, T-03 to T-05) + and `tests/devenv/unit.sh`. That run is the first evidence that this + implementation, and not only the spike, isolates. +3. Performance taken from that run (L-16 and the job times), not from the + estimate. + +## Rollback + +Revert the commits that introduced this change on their branch. The Linux +path then refuses, or uses the previous mechanism of the reverted-to +commit. Profiles never outlive the runner's VM, so nothing persists +between jobs. diff --git a/docs/dev-environment.md b/docs/dev-environment.md index 8c5281a..3ab2bbe 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -413,6 +413,943 @@ $ DEVENV_TEST_PORT_BASE=18080 tests/devenv/integration.sh $ make test-app # application: the four suites below ``` +### Isolation from Docker + +`tests/devenv/unit.sh` and the guard regression suites in +`scripts/ci/tests/` run code under test, including a modified or mutated +copy of it (`DEVENV=`, `GUARD=`, `ADAPTED=` or an edited file). That code +runs with the caller's user ID and may be any shell, Python or program on +the machine: it may fork, daemonise, write files, open sockets and send +signals. Each suite therefore re-executes itself under +`scripts/ci/no-docker.sh`. The isolation must keep the code away from: + +- the container engines and their objects (images, containers, volumes, + networks, build cache), including those of other projects; +- the user's environment: files in `$HOME` and in the checkout, shell and + tool configuration, user-session IPC and the user's other processes; +- later steps of the same CI job. + +The person or job that starts a suite is trusted. The checks of the +environment described below prevent accidental use where the isolation is +not supported; they are not a boundary against the caller. Kernel and +sandbox vulnerabilities and CPU or memory exhaustion are out of scope. + +Supported platforms: + +| Platform | Support | Mechanism | +|---|---|---| +| macOS developer machine | supported | `sandbox-exec` profile | +| Linux on a GitHub-hosted Actions runner | supported | `sudo -n unshare` namespaces and an AppArmor profile per launch | +| Linux anywhere else (developer machines, self-hosted runners, containers) | refused (exit 125): "Linux is supported on GitHub-hosted CI runners only" | — | +| other systems | refused (exit 125) | — | +| any platform as uid 0 | refused (exit 125) | — | + +Each property below is enforced by the sandbox profile or the kernel. Before +the command starts, `scripts/ci/no-docker-probe.py` checks from inside the +ones each platform's section lists as checked; on macOS the others rest on +`sandbox-exec` accepting the profile and on the regression tests. If any +check fails, a tool is missing, a mount fails, the profile is rejected or +the command was not started inside the isolation, the command does not run +and the exit status is 125. On Linux the status is 125 as well when the +launch's AppArmor profile cannot be shown safe to remove afterwards (it +then stays loaded; see "Linux"). + +On both platforms the variables that select an engine (`DOCKER_*`, +`CONTAINER_HOST`, `PODMAN_*`, `COMPOSE_*` and similar) are removed, +`DOCKER_CONFIG` is an empty directory and `$TMPDIR` is a directory of the +run. The probe then asks every endpoint it can find whether a container +engine answers there: the usual socket paths, the Docker contexts, +`DOCKER_HOST`, sockets under `/run`, `/var/run`, `/tmp`, `$TMPDIR` and +`~/.docker`, ports 2375 and 2376 and every listening TCP port. The walk +of those directories stops at 20,000 entries. On Linux that refuses the +run. On macOS, where every Unix socket connect is denied anyway, the probe +prints an INFO line instead. + +#### macOS + +`$TMPDIR` must be the per-user temporary directory that macOS creates, +`/private/var/folders///T`, owned by the user with mode 0700. If +`TMPDIR` is unset, `getconf DARWIN_USER_TEMP_DIR` is used. Any other +directory, such as `/tmp`, `/private/var/tmp`, `/`, Homebrew's prefix or a +directory of the user's own, is refused. Each run gets a fresh directory +there, mode 0700. The command's `$TMPDIR` is inside it. + +The `sandbox-exec` profile denies: + +- connecting to any Unix socket. The probe checks this with a socket of its + own: EPERM; +- TCP beyond loopback, and to loopback ports 22, 2375 and 2376. The probe + checks 2375 and a non-loopback address; +- TCP to every port that was listening when the run started, on any address + (a development server, an IDE, a published container port or a socket + proxy). The list comes from `netstat`; if `netstat` fails, prints no + column header or prints a LISTEN row that cannot be parsed, the run is + refused. The probe connects to each held port; +- file writes anywhere except in the run's directory, `/dev/null`, + `/dev/zero`, `/dev/tty`, `/dev/dtracehelper` and `/dev/fd/*`. Nothing in + `/tmp` or `/private/var/tmp` is writable, so no file another process + keeps there can be changed, replaced or created from inside. The probe + checks that `$HOME`, the user's temporary directory, `/tmp`, + `/private/var/tmp` and an `sh-thd-*` file in `/tmp` are not writable and + that `$TMPDIR` is. This `/dev` list was measured with bash 3.2, python3, + `mktemp`, `/usr/bin/git`, `tests/devenv/unit.sh` and the guard suites. + Writing to `/dev/random`, `/dev/urandom` and other terminals' `/dev/ttys*` + is denied. `/usr/bin/git` reports that it cannot write the `xcrun_db` + cache in the user's temporary directory and still works. That cache is + read by `xcrun` outside the sandbox, so it stays write-denied; +- signals to any process outside the sandbox. Processes of the run can + signal each other, including process groups. The probe sends signal 0 to + `no-docker.sh` outside: EPERM; +- debugging any process (`system-debug`): `ptrace(PT_ATTACHEXC)` fails with + EPERM, even against a process of the user that allows debuggers + (`get-task-allow`). Measured: without this rule the attach succeeds, and + a debugger could then stop, signal or end that process; +- the task ports of processes outside the sandbox (control, inspect, read + and name ports), which debuggers and readers such as `vmmap` and `sample` + use. Measured: without the rule they read the memory map of such a + process from inside; with it they are refused, while they still work on + the command's own processes and `ps` still lists every process; +- reading the devices of other terminals (`/dev/ttys*`), which would let + the command read what is typed in them. The command keeps its own + terminal through `/dev/tty` and the descriptors it inherits; +- preference writes through `cfprefsd` and `TIOCSTI` on the terminal; +- `open`, `osascript`, `launchctl` and `ssh`, job submission to launchd, + Launch Services and Apple Events. Measured with a re-signed copy of + `launchctl` inside: `submit` and `bootstrap` of a job and `setenv` in the + user's launchd domain all fail, and no job or variable appears; +- lookups of the services that register login items and background tasks + (`com.apple.xpc.smd`, `com.apple.backgroundtaskmanagement*`, + `com.apple.coreservices.sharedfilelistd*`) and of Docker Desktop's own + services (`com.docker.*`, `com.electron.dockerdesktop.*`). + +bash 3.2, which is also `/bin/sh`, keeps each here-document and +here-string in a temporary file, ignoring `$TMPDIR`: in `/private/var/tmp`, +or in `/tmp` where `/var/tmp` is not writable, as inside a sandbox. Neither +is writable inside, so a here-document fails there ("cannot create temp +file for here document"). Measured: bash then falls back to the working +directory, so one works only while the working directory is in the run's +directory. The code that runs inside (`bin/devenv`, `tests/devenv/unit.sh`, +the guard suites and the scripts they run) therefore uses none; T-20 +checks this statically. + +When the command ends, the processes it left in the sandbox are ended: a +plain background job, a daemon after `setsid` and a double fork, a +`posix_spawn` child in a session of its own, a process group of its own. +`no-docker.sh` hands its process to a supervisor in +`scripts/ci/no-docker-probe.py`, which starts `sandbox-exec`, waits for it +and then ends what is left. It finds those processes by a marker: the +profile denies reading file `a` of the run's marker directory, in `$TMPDIR` +outside the run's directory, and leaves file `b` next to it alone. From +outside, `sandbox_check()` asks how a process's sandbox would decide; a +process is in the run's sandbox exactly when it is sandboxed, denied `a` and +allowed `b`. Measured: an unsandboxed process is allowed both, and the +files must exist, or every process is "denied". A process cannot leave its +sandbox, and every process it starts inherits it, however it detaches. + +The run is reported done only when it is shown that none of its processes +runs any more. The supervisor takes one copy of the kernel's process list +at a time (the kernel copies it under one lock, so it has every process of +that instant, zombies included) and checks each process in it, by its +unique id before and after the check: one of the run's is sent SIGKILL at +once. A copy shows the run has ended when every process in it is shown not +to be one of the run's, or was already dead (a zombie) before that copy was +taken. Every process started later descends from one in the copy (the +kernel lists a child before it can run, and its parent until it is +reaped), so none of them can be one of the run's either. The one process +that becomes one of the run's after it could be checked is the run's own +`sandbox-exec` before it applies the profile: the supervisor waits for it +first, and when a run is checked whose owner is gone, a process that owner +started keeps every copy open. The argument relies on a pid not being +reused while one copy is checked (milliseconds; the kernel's counter wraps +at 99999). A process that ends before it can be +checked leaves that copy open, as it may have started one of the run's +first; the supervisor then takes the next copy. Measured: a run without +leftovers is shown done in about 0.07 seconds, a run whose daemon must be +ended in about 0.4 seconds. + +If that cannot be shown within 10 seconds, the run fails with status 125; +it is never reported done. Measured: a process that forks a new one and +ends, over and over (thousands of times per second), makes every copy +open: the run fails, although that process is not found and keeps running, +still in the sandbox with every limit above. The run's marker directory is +then kept, and no new run starts until it is shown that none of its +processes runs any more (then that directory is removed). The supervisor +needs no new process for any of this, so a command that uses up the +processes it may start cannot stop it. SIGINT, SIGTERM, SIGHUP or SIGQUIT +delivered to `no-docker.sh` end the command and everything it left in the +same way, and the status is 128 plus the signal number; a signal the +caller ignores stays ignored. The probe checks from inside that the +sandbox carries the run's marker. + +SIGKILL of `no-docker.sh` (or a crash of the supervisor) cannot be handled: +what the command left keeps running, still inside the sandbox with every +limit above. The marker directory records the run's owner (the process +that becomes its supervisor). Before each run, `no-docker.sh` looks at the +marker directories whose owner is gone; it removes one once it is shown +that none of its processes runs, and otherwise refuses to start, naming +them. It never ends anything there. Runs in progress elsewhere (their +owner lives) are left alone. + +Setuid programs cannot run in the sandbox, so `ps` is an ad-hoc signed copy +and `sudo` is unavailable. The U-16 check that another local user cannot +read a secret therefore prints INFO and is not counted; the U-16 mode checks +still run. macOS applies a sandbox inside another only if the profiles are +identical, so the suites cannot run inside a different sandbox: they are +refused. + +#### Linux (GitHub-hosted runners) + +`no-docker.sh` refuses unless: + +- the caller is not uid 0 (real or effective). This is checked before + anything else; +- `GITHUB_ACTIONS=true` and `RUNNER_ENVIRONMENT=github-hosted`; +- a `Runner.Worker` process (by `comm` and executable name) is an ancestor + of the caller. An unreadable ancestor counts as a refusal; +- the caller is in the initial pid namespace (`pid:[4026531836]`), so the + namespace made below it is shown to descend from the caller's; +- `$GITHUB_WORKSPACE` and `$RUNNER_TEMP` are directories under `$HOME`; +- the caller's stdout and stderr are open and neither a socket nor a + terminal. The command's output reaches them through pipes of + `no-docker.sh` (below); a socket or terminal there would be a channel out + of the isolation. stdin is not used; +- `sudo -n` works, and `timeout`, `unshare`, `setpriv`, `setsid`, `ip`, + `mount`, `find`, `stat`, `python3`, `od`, `sync`, `/usr/bin/aa-exec` and + `/usr/bin/python3` exist; +- `scripts/ci/no-docker-aa.py preflight` (as root) shows that AppArmor can + confine the launch: it is enabled, its global mode is `enforce`, the + kernel mediates `af_unix` (`features/network/af_unix`) and + `apparmor_parser` and `aa-exec` are present. It also records, without + gating, the parser version, the network feature directories, + `kernel.apparmor_restrict_unprivileged_unconfined` and + `..._userns`, `core_pattern` and sudo's `env_reset` and `closefrom`. + +**The boundary** (decision: +[ADR-023](architecture/adr/ADR-023-linux-test-isolation-apparmor.md), +proposed). Host Unix sockets are not listed any more: the kernel refuses +them. Each launch loads an AppArmor profile of its own, +`flowcrm-nd-<32 random hex digits>`, from the pinned template +`scripts/ci/no-docker/apparmor.profile`: + +``` +abi , +profile @NAME@ flags=(enforce) { + file, + unix, + network, + signal, + ptrace (read, readby, trace, tracedby) peer=@NAME@, + deny capability, + deny mount, + deny umount, + deny pivot_root, + deny network vsock, +} +``` + +The profile has no `attach_disconnected`. A process confined by it, in a +private mount namespace, that connects or sends to a pathname socket bound +outside that namespace is refused with `EACCES`: the socket's bound path +lies on a mount the process's namespace does not contain, so AppArmor sees +it as disconnected. That holds whatever path, alias, bind mount or +unlistable directory leads to the socket and whenever it was bound, for +stream and datagram sockets alike. Sockets bound inside the namespace stay +usable. The denial is Ubuntu's AppArmor code (`RULE_MEDIATES_UNIX`), shown +on `7.0.0-1017-azure` with parser 4.0.1 (spike run 41209833578); on a +kernel where it does not hold, the fence below refuses every launch. + +`no-docker-aa.py` (as root through `sudo -n timeout N`) loads the profile +only from the template whose sha256 it pins, adds it without replacing, +refuses a name already loaded, and checks that it is listed exactly once as +`NAME (enforce)`; it records the kernel's hash of the profile when the +kernel exports one. Its only effects are that load, the removal of the +launch's own profile, SIGKILL of the launch's pid 1 (below) and files in +the launch's control directory. + +**The launch**, in order: + +1. A control directory and a fence directory in `$RUNNER_TEMP` + (`no-docker-ctl.XXXXXX`, `no-docker-fence.XXXXXX`, mode 0700) and the + profile name. The launch's state is written to the control directory + (`loading`), atomically (write, `sync`, rename) and read back. +2. The fence: two sockets bound outside by listeners of `no-docker.sh` (a + stream and a datagram socket, children of its shell with + `PR_SET_PDEATHSIG`), shown reachable from outside. +3. The profile is loaded (the helper is limited to 20 s; `no-docker.sh` + waits for it up to 25 s). Whether `sudo` still runs is read from + `/proc` (pid and start time), never with `kill -0`. +4. `::stop-commands::` is printed on stdout and stderr; no + argument or variable passes the token inside (see RR-10). Two relays, `cat` processes of + `no-docker.sh`, will copy the command's stdout and stderr to the + caller's. The shell's own descriptors are then listed (by a child, into + the control directory): besides 0 to 2, the relays' write ends and the + script, other copies of a relay pipe are closed, and anything else + refuses the launch. +5. The state `entering` is written and read back; if it cannot be, + nothing is started. Then + `sudo -n unshare --net --mount --pid --fork --kill-child --mount-proc --ipc --propagation private` + starts the namespace, with stdin `/dev/null` and the relays as stdout + and stderr; the state becomes `entered spid=`. +6. The namespace's pid 1 is a root shell that does not exec. Its first + action, before anything untrusted exists, records its pid and mount + namespaces in `CTLDIR/ns` (`pid: mnt:`, root-owned, mode + 0444, written, synced and renamed). Then, as before: loopback up; empty + tmpfs over `/run` (and `/var/run` when that is not a symbolic link), + `/tmp`, `/var/tmp` and `/dev/shm`; a fresh mqueue on `/dev/mqueue`; a + read-only tmpfs over `/sys/fs/cgroup`; the run's directory moved back + into the new tmpfs; every other mount reachable by path remounted + read-only. This covers `$HOME` with the workspace, `$RUNNER_TEMP` with + the control and fence directories, `$GITHUB_ENV`, `$GITHUB_PATH`, + `~/.docker` and `/opt/hostedtoolcache`. +7. A child of pid 1 runs `no-docker-probe.py --root-exec` as root. It + closes every descriptor above 2 up to the hard `RLIMIT_NOFILE`, shows + that none is left, opens `/dev/null` there as fd 0 and logs + `closed: ...; remaining: 0 1 2; stdin char 1:3`. It then execs `setpriv` + (the caller's uid and gid, no supplementary groups, `no_new_privs`, + empty inheritable and bounding capability sets), `aa-exec -p NAME`, + which attaches the profile at the next exec, `setsid --wait`, `env -i` + with the allowlist of variables (`PATH`, `HOME`, `USER`, `LOGNAME`, + `SHELL`, `LANG`, `LANGUAGE`, `LC_*`, `TERM`, `TZ`, `CI`, `DEVENV*`, + `GUARD*`, `ADAPTED`, `HARNESS`, `FLOWCRM_*` and the isolation's own + `NO_DOCKER_*`) and the gate. +8. The gate, `no-docker-probe.py --gate-exec`, confined, as the caller, + checks in this order: + - **G1:** the rules below. + - **G2:** the namespace record names its own pid namespace, and the + control and fence directories are read-only. + - **G3:** it, and every other process but the root pid 1, carry exactly + `NAME (enforce)`. A stacked or complain-mode label, or an unconfined + one, refuses. + - **G4:** the two fence sockets, which it owns and which file + permissions let it reach (shown with `faccessat(AT_EACCESS)`, which + AppArmor does not consult), refuse `connect` and `sendto` with + `EACCES` exactly. Any other result refuses: the refusal must come + from the profile. + - **G5:** a socket it binds in its private `/tmp` connects. + - **G6:** both are closed. + - **G7:** only if everything so far passed: the writes and the + engine check of the rules below. + - **G8, last:** its descriptors are exactly fd 0 `/dev/null` and pipes + on fds 1 and 2. + + It reports every finding on stderr. Only if all passed does it reset + every signal to its default disposition, clear the signal mask, record + the start and exec the command in its own process, so the process + checked is the process that runs. +9. `no-docker.sh` waits for `sudo`, then cleans up (below). + +The rules (G1, and G7's writes), checked from inside: + +- no uid is 0; every capability set is empty; `NoNewPrivs` is 1; there are + no supplementary groups; the four gids are equal and not a system group + (above 999, as Debian and Ubuntu number them: the Docker group, `disk`, + `adm` and the like are below); +- the network, mount, pid and IPC namespaces differ from the caller's; +- the caller's pid namespace is the initial one (`pid:[4026531836]`, its + fixed inode number) and the process's is not, so it is below the + caller's: every pid namespace descends from the initial one; +- `/proc` is of the process's own pid namespace: `NSpid` is its pid alone + (a procfs shows a task's pids from its own namespace down, so a `/proc` + of an ancestor namespace would show two or more), and `/proc/self` is + its pid. `/proc` has no `hidepid` or `subset` option; +- pid 1 is the root shell of the isolation and every other visible process + has the caller's uid. A process whose status cannot be read, other than + one that just ended, counts as foreign, and pid 1 must be read. None is + a runner, init or engine process, so `/proc//root` and `nsenter` + have no target outside; +- the process is in a session of its own without a controlling terminal, + so `TIOCSTI` has no target; its standard descriptors are pipes or + `/dev/null`; +- loopback is the only interface; +- `/tmp`, `/var/tmp`, `/dev/shm` and `/run` are tmpfs, and `/dev/mqueue` an + mqueue, on devices other than the caller's. `/sys/fs/cgroup` is a + read-only tmpfs, and no cgroup file is writable; +- every mount reachable by path is read-only, except the private file + systems above, the run's directory, `/proc` and `/dev`. This is read from + `/proc/self/mountinfo` and the mount ID of each path. Creating a file in + `$HOME`, `$GITHUB_WORKSPACE` and `$RUNNER_TEMP` must fail as well; +- the working directory is reachable by its path; +- no other `proc` file system and no `nsfs` mount is reachable by path; +- there is no socket in the private directories. + +A suite that re-executes itself inside (`no-docker.sh --check`, or a nested +`no-docker.sh`) is checked against G1, its own confinement (G3) and, if +those pass, G7 and the engine check; its descriptors are only reported. +This is not a boundary: the gate of the launch is. + +**Cleanup.** One function, run once on every way out (the normal end, a +refusal, SIGINT, SIGTERM, SIGHUP), touches only what this launch recorded +as started, with every wait bounded (112 s in all): + +1. If the namespace may still run, `no-docker-aa.py kill-init` finds its + pid 1 from the launcher's process tree, holds it by a pidfd, checks its + start time around reading its namespace, records `CTLDIR/ns` if pid 1 + had not, and sends it SIGKILL. When pid 1 exits, the kernel kills + every other process of the namespace. Nothing else is ever signalled. + After a normal end `sudo` has been waited for, so this step is skipped. + If the fork of `sudo` had not happened, the state is set to + `not-forked`. +2. The relays end once every writer is gone; only then is the token that + resumes workflow commands printed, on a line of its own (after an empty + line), so the command's output can not reopen them early. +3. The fence listeners are ended by pid and start time, and the fence + directory is removed; a failed removal is a cleanup failure. +4. A load still in flight is waited for. +5. `no-docker-aa.py release` removes the profile only when: + - **P0:** `CTLDIR/ns` is a root-owned file of mode 0444 in the exact + format, naming a pid namespace that is not the host's; + - **P1:** no task is in that pid namespace, except its init as a + single-threaded zombie; + - **P2:** the profile is listed once, in enforce mode, with its + load-time hash; + - two complete sweeps of every task's `attr/apparmor/current` find no + label containing its name. + + States `loading` and `not-forked` without `CTLDIR/ns` mean no namespace + was made: the profile is then removed after P2 and the sweeps, or + nothing is done when it was never loaded. Any error other than a task + that ended counts as unknown, and unknown keeps the profile loaded: + removing a profile unconfines any process that still carries it. +6. If the profile cannot be shown safe to remove, it stays loaded, the + control directory and the run directory are kept and their paths + printed (`no-docker: refused: profile NAME still loaded (...)`), `diag` + reports what is known, and the run exits 125 even if the command + succeeded. After a failure of an earlier cleanup step (such as a fence + directory that could not be removed) the control directory is kept as + well. Otherwise the control directory is removed, and then the run + directory (one `no-docker.sh` made, never a `--work` directory). A + failed removal of either is a cleanup failure; what was removed before + it stays removed (a run directory that cannot be removed leaves no + control directory). +7. The status, in this order: + - a cleanup failure gives 125; + - the negative control gives 125; + - a signal gives 128 plus the signal (130, 143, 129); + - a refusal gives 125; + - a command that did not start gives 125; + - otherwise, the command's own status. + +A `::notice` on stderr gives the time of each phase of the launch; the gate +prints its own. + +**Negative control.** `no-docker.sh --negative-control COMMAND` (GitHub-hosted +Linux, first level only; refused nested and elsewhere) loads the +negative template `scripts/ci/tests/no-docker/apparmor-negative.profile` +(the same rules with `attach_disconnected`) as `flowcrm-nd-neg-`. The +gate must then find the fence reachable and refuse. COMMAND is replaced by +`/usr/bin/true` before anything starts and the status is always 125. + +**Interruption.** SIGINT, SIGTERM or SIGHUP delivered to `no-docker.sh` +itself (Ctrl-C in a terminal, or a caller that forwards the signal) run the +cleanup above. A cancelled or timed-out GitHub step does not reach it: the +runner signals only the step's shell, and later sends SIGKILL to the +process tree. The namespace then keeps running, confined, with read-only +mounts and no network, and the profile stays loaded, until the runner's VM +is discarded (see the residual risks). + +The probe also prints the uids and gids of the command, and `no-docker.sh` +the caller's primary group with its name and the group of +`/var/run/docker.sock`. The command keeps the caller's primary group, +which must not be a system group (above). + +**Status of the evidence.** This Linux path has run on a GitHub-hosted +runner. CI run +[42049053505](https://github.com/knittingcode/flowcrm/actions/runs/42049053505) +of 2026-11-05, a push of preview commit +`7e7d4d38d054caeb5756bc785a6e73cab42548ea`, succeeded and executed all six +protected suites, among them `scripts/ci/tests/no-docker.test.sh` (182 of +182) and `tests/devenv/unit.sh` (341 of 341). The earlier spike runs +41200868832 (inconclusive) and 41209833578 (GO) showed the kernel mechanism +only, not this launcher. + +That run is evidence from a push of a preview branch. It does not replace +the required `pull_request` checks of the final integration revision, which +must pass on that revision. It is not a release approval. It does not show +that the images have no vulnerabilities: the container scan passed under its +exception policy. It does not show that nothing was left behind: the +workflow takes no final inventory of the runner, and the removal of the scan +cache reported permission errors in that run. The accepted residual risks +below still apply. + +#### Run directory (`--work DIR`) + +`no-docker.sh --work DIR COMMAND` uses `DIR` as the run's directory +instead of a temporary one, and keeps it afterwards. It is the command's +only writable directory and holds its `$TMPDIR`. + +`DIR` must: + +- be given as an absolute, canonical path with no symbolic link in it; +- be a directory of the caller with mode 0700; +- be empty; +- not be or contain `$HOME`; +- be under `$TMPDIR` or `/private/var/tmp` on macOS, and under `/var/tmp` + on Linux. + +Otherwise the run is refused. `tests/devenv/unit.sh` passes +`NO_DOCKER_WORK=` on as `--work`. Whatever the command leaves in +`DIR` is untrusted data: a later reader must not follow symbolic links, +must accept regular files only, must limit their size and must not execute +them. + +#### Accepted residual risks + +- **SIGKILL of `no-docker.sh` (macOS), or an ending that cannot be shown.** + The command and what it left keep running in the sandbox, with every + limit above, until someone ends them (`no-docker-probe.py --end `, or by pid); meanwhile no new isolated run starts. A process + that keeps forking and ending is not found at all; it runs until it stops + by itself, the user ends it or logs out. Until then the channels the + sandbox leaves open (loopback ports, the pasteboard, reading the user's + files and process information) stay available to it, without a time + limit. A marker directory of an earlier version (without the owner + record) is checked like one whose owner is gone once it is a minute old; + processes of such a version whose directory was already removed are not + recognised: end them by pid after updating. +- **Process identity (macOS).** A process is signalled only after its + unique id and membership were read again, microseconds before. A pid + reused within that window by a process outside could still be ended: + an availability risk, not a way out. Reusing a pid needs the kernel's + counter to wrap (pids up to 99999). +- **Late listeners.** A loopback port that starts listening after the run + started (on macOS) is reachable, as are the ports the command opens + itself. This includes a VM's port forward that starts later. The probe + asks each listening port whether it is an engine only at the start. The + other way round, a loopback port the command opens during the run can be + reached by programs outside, which may take its answers for those of a + local service (for example a readiness check of `bin/devenv` run at the + same time). +- **Login items and launchd (macOS).** The services that register login + items and background tasks are denied, and launchd jobs and `setenv` + through a re-signed `launchctl` fail (measured). Services that launchd + starts on the command's behalf run outside the sandbox; they are limited + by the lookups and Apple Events denied above, not by the supervisor. +- **Terminals and the pasteboard (macOS).** Reading and writing other + terminals' devices are denied. A process of the command can still make + itself the foreground process group of its own terminal while it runs. + The pasteboard (`pbcopy`) is not denied; using it needs a person to + paste. +- **Reading other processes (macOS).** The environment and arguments of the + user's other processes are readable (`ps eww`), as are the user's files. +- **Writes through other processes (macOS).** The file-write limit covers + the command's own writes. Writes made for it by system services are + limited by the launchd, Launch Services, Apple Events, preference and + login-item denies above, not by the file rule. Keychain items are data, + not code. +- **The kernel and the image (Linux, RR-02, RR-03).** The denial is + Ubuntu's AppArmor code. A kernel, parser or runner image change can + disable it; the fence then refuses every launch (a red run, never an + unisolated one). Attaching the profile after `setpriv` needs + `kernel.apparmor_restrict_unprivileged_unconfined=0` (measured); with 1 + the label is stacked and every launch refuses. +- **io_uring, rename and hard links (Linux, RR-04).** A connect through + io_uring, or a host socket renamed or hard-linked into the namespace, + passes the same AppArmor hooks; that rests on reading the kernel source, + not on a test. +- **Paths made by root (Linux, RR-05).** A root process outside that binds + a socket through `/proc//root/...` would make a + path inside the namespace. No trusted component does this. +- **A late start (Linux, RR-06).** A signal while the namespace starts can + leave a chain that reaches pid 1 only after cleanup ended. Its gate then + finds no fence and refuses, so its command never starts; the profile + stays loaded (125) and can be released once the chain has ended. +- **Attaching from outside (Linux, RR-07).** Only an unconfined trusted + process can attach the launch's profile (the command cannot: + `change_profile` is denied). One that does so after the last sweep and + before the removal is returned to unconfined by the removal: no boundary + is crossed. One present at sweep time keeps the profile loaded. +- **Audit records (Linux, RR-08)** may be lost; nothing relies on them. +- **The launch chain (Linux, RR-09).** Descriptor hygiene relies on the + chain staying as designed; the gate checks the process that becomes the + command on every launch. +- **Annotation order and the stop token (Linux, RR-10).** Output the + command wrote earlier on stderr may reach the runner after the resume + token on stdout (as before); it can only add annotations. The token is + not passed inside, but when the caller sends stdout or stderr to a file + the command can read (in `$HOME`, the workspace or `$RUNNER_TEMP`), the + command can read it there and resume workflow commands itself: give it a + pipe, or a file elsewhere. No step of this repository's workflow uses step outputs + or unsecure commands, and `$GITHUB_ENV` and `$GITHUB_OUTPUT` are + read-only inside. +- **Untrapped signals (Linux, RR-11).** A signal `no-docker.sh` does not + trap (such as SIGQUIT) ends it without cleanup, as a cancellation does + (below). +- **A reused namespace inode (Linux, RR-12).** If the recorded pid + namespace inode is reused by another namespace during the release, P1 + only refuses more: a false 125, never a removal with a carrier left. +- **No namespace record (Linux, RR-13).** If pid 1 fails (for instance its + write of `CTLDIR/ns`) or `sudo`/`unshare` fails before forking it, and + `kill-init` never found it, the profile can never be shown safe to + remove: it stays loaded with no process carrying it (125) until the VM is + discarded. +- **Shared keyring and readable files (Linux).** The per-user kernel + keyring is shared. Files such as `~/.docker/config.json` stay readable, + but there is no network to use them on. +- **Orphan cleanup after a kill (Linux).** `env -i` drops + `RUNNER_TRACKING_ID`, so the runner's end-of-job cleanup cannot find + processes of the namespace. They are killed with pid 1 instead, on a + normal end and when SIGINT, SIGTERM or SIGHUP is delivered to + `no-docker.sh` itself (see above). +- **Cancelled or timed-out CI steps (Linux, RR-01).** GitHub signals only + the step's shell and later sends SIGKILL to the process tree. Neither + reaches the trap in `no-docker.sh`, so the namespace and anything left in + it keep running until the runner's VM is discarded, and the launch's + profile stays loaded. Nothing removes it, so they stay confined, with + read-only mounts, no network, no capabilities and no view of other + processes. This is reported as a residual, never as a cleanup. Only the + checkout's post-job step follows. +- **CI coverage when a scan fails.** Isolated steps run last in their job + (see below). When the container scan fails, the Dev environment job stops + before `tests/devenv/unit.sh`, so the allocator unit tests do not run in + that CI run. + +#### CI + +The isolated steps are the last steps of each required job: + +- **Repository guard:** the trusted guard scripts, ShellCheck and + actionlint run first; then the guard regression suites and + `scripts/ci/tests/no-docker.test.sh`. +- **Dev environment:** `tests/devenv/unit.sh` runs after the container + scan. + +After them, only the checkout action's post-job step runs. During the +isolated steps on Linux it reads nothing that the code under test could +have written: the workspace and `$HOME` are read-only, and `/tmp` is +private. + +`scripts/ci/tests/ci-isolation-order.py` checks the order statically: + +- after the first isolated step of a job, every following step is + isolated. A step is isolated when every command it runs (directly or + through a `make` target) is `no-docker.sh`, its test harness + `no-docker.test.sh`, or a script with the re-exec stanza. A step that + mixes such commands with others fails; +- the workflow is in a strict subset of YAML, read without PyYAML, which the + check cannot assume on the runner. Any other spelling (`env :`, a bare `-`, an + anchor, a merge key) fails, because GitHub may read it differently; +- the workflow, its jobs and its steps have only known keys, so `needs:` and + `strategy:` fail, and the workflow runs for every pull request with no path + filter; +- the required jobs exist once and run exactly their six suites, in order; +- no step writes to `$GITHUB_ENV` or `$GITHUB_PATH`, in its text or in a script it + runs. A computed name is not seen. The only action allowed is `actions/checkout`, + pinned to a commit, with `persist-credentials: false`, because another could export + settings from text the check does not read. No `run:` text has a `${{ }}` + expression, which GitHub expands before the shell; +- nothing but comments, `set` options and the assignment of `$here` comes before + the re-exec stanza of a script that counts as isolated; +- no job or step has an `if:` condition or a `continue-on-error:` key; +- every isolated step of the required jobs is exactly one command through + `scripts/ci/fdnorm.py`, and that file exists; +- no isolated step has a key other than `name`, `run` and `timeout-minutes`, + and no `env` or `defaults` reach it from its job or the workflow, because + `shell`, `env` and `working-directory` change how an exact `run:` line behaves; +- every other file that mentions `no-docker.sh` has the recognised re-exec + stanza. + +GitHub-hosted steps were seen to inherit open pipes above fd 2 (fds 142 and +145 in run 41636839721; which process opened them is not known). The launcher +refuses any descriptor it did not open itself, so all six isolated steps run +`python3 -I scripts/ci/fdnorm.py -- PATH`. The wrapper closes every descriptor +above 2, checks that only 0, 1 and 2 remain unchanged, reports what it closed +to fd 2 as `ci-fd:` lines, and then becomes `PATH` in the same process. If it +cannot prove that state it exits 2 and `PATH` does not run. It does not change +the launcher or what the launcher checks, and it does not isolate anything: the +order check treats the wrapper as transparent only in that exact form, so +`PATH` must already be isolated and must name a directory. Arguments, +redirections, other interpreters, variables and wrapped commands such as +`docker` are not isolated. + +#### Regression tests + +`scripts/ci/tests/no-docker.test.sh` runs a mutated `bin/devenv` whose +commands reach Docker in more than 20 ways, both as `DEVENV` and edited +into the file. The mutant only reads (`docker version`, `GET /version`); +its text is pinned by sha256 and checked before each use. Its target is a +canary engine of the test, which answers like Docker and records every +request. + +- **T-03 and T-04:** on macOS, `unit.sh` runs on the host as a developer + would run it. The mutant must run to its end without a request reaching + the canary. On Linux, `unit.sh` runs inside the isolation, where the + canary is a reachable engine, so it must refuse. +- **T-05 (control):** a copy of `unit.sh` without the re-execution, behind + an outer guard. The mutant must reach the canary. On macOS, that guard + (also used for every case with `HARNESS=`) has the same write policy and + denies as `no-docker.sh`. Only the case's own directories are writable, + and only the canary's socket is reachable. +- Files that code under test could write are read only when they are + regular files, so a planted FIFO or link cannot hang or redirect the + test. +- **T-14 and T-23:** the helpers that the test runs outside any isolation + (`canary.py`, `drive.sh`) and the mutant's text are tracked files in + `scripts/ci/tests/no-docker/`. Their sha256 values are constants in the + test, set when they were reviewed, and every copy is checked against + them immediately before each run outside, so a copy changed at any time + is never run. The copies are kept outside every directory that code + under test can write. T-14 checks that a changed copy is detected and not + run, and T-23 that the values are constants. + +The other cases run plain commands that only print, connect, write markers +in the test's directories or send signal 0 or TERM to the test's own +`sleep`: + +- **T-00:** the read-only lint of the mutant. +- **T-01 and T-02:** `--check`, and a claimed but unproven isolation. +- **T-07:** loopback ports that were already listening are unreachable, + while ports opened inside are reachable. +- **T-06, T-08 to T-15, T-21, T-22 and T-24 to T-35, macOS only:** + - **T-06 and T-11:** the write limits, including that a bash 3.2 + here-document fails; + - **T-08:** the complete port list; + - **T-09 and T-12:** the `TMPDIR` allowlist; + - **T-10:** preferences and `TIOCSTI`; + - **T-13:** a canary outside cannot be signalled (EPERM) and survives, + while the command's own child and process group can be signalled; + - **T-14:** integrity of the helpers; + - **T-15:** `--work`, accepted and refused cases; + - **T-21:** canary files named like bash 3.2 here-document files in + `/tmp` and `/private/var/tmp` (random names, created exclusively, read + and removed only while they are the files created) cannot be opened for + writing, truncated, removed, renamed, replaced or chmodded, and none can + be created; + - **T-22:** a file outside the run's directory cannot be changed through + a hard link, a symbolic link, a rename over it or an exchange; + - **T-24:** four processes the command leaves (a background child, a + `setsid` daemon after a double fork, a `posix_spawn` child in a session + of its own, a process group of its own) are all ended, after a normal + end, a failed command and a command killed by SIGKILL; + - **T-25:** SIGINT, SIGTERM and SIGHUP to `no-docker.sh` end the command + and those four processes at once (status 128 plus the signal); + - **T-26:** after SIGKILL of `no-docker.sh`, its leftovers keep a new run + from starting (125) until they are ended through the run's marker + directory, whose owner is gone; + - **T-27:** when the probe cannot make the markers, look for leftovers or + supervise, the command is not run; + - **T-28:** a debugger attach from inside to a test canary outside that + allows debuggers fails, while it works from outside; + - **T-30:** a process that keeps forking and ending is never reported as + ended; new runs refuse while it runs and start after it ended; + - **T-31:** two runs at the same time each end their own processes and + do not refuse each other; + - **T-32:** processes that start and end outside meanwhile do not make + the ending fail; + - **T-33:** an ending that cannot be shown fails the run (125), ends the + supervisor's own child and keeps the marker directory; + - **T-34:** SIGTERM at several moments while the command starts processes: + every run ends in time and none of its processes survives; + - **T-35:** a zombie outside the sandbox (a child of this suite that has + ended) does not make the ending of a run unprovable; + - **T-29:** another terminal's device cannot be read from inside, while + it can from outside. +- **T-16 to T-18, Linux only:** + - **T-16:** a canary inside sees `NSpid` as its pid alone, is in a pid + namespace other than the caller's (the initial one), sees only the + namespace's processes (an unreadable one counts as foreign) through a + `/proc` without `hidepid`, cannot read `/proc/1/root`, is confined by + exactly the launch's profile, has only descriptors 0 to 2 and finds + the launch's fence refused. Every host socket the suite gathers outside + (the kernel's list and `$HOME`, plus one of its own in `$RUNNER_TEMP`) + is classified: reachable outside and visible inside, it must be refused + with `EACCES`; under a directory the isolation hides, it is not found; + not reachable outside, it is only reported; none is ever reached + inside. It gets EROFS in `$HOME`, the workspace, `$RUNNER_TEMP` and the + tool cache, and sees a private `/tmp` and `/dev/shm`, read-only mounts, + masked cgroups, private IPC and loopback only. Every check must print + exactly one ok line; + - **T-17:** a `setsid` daemon, checked to be running before the command + returns, does not outlive the command, neither when the command ends + nor when SIGTERM is delivered to `no-docker.sh` itself (status 143, + within 150 s). A cancelled GitHub step does not signal `no-docker.sh`, + so that case is not covered; + - **T-18:** uid 0 (a real `sudo` run), missing or other runner variables, + a workspace outside `$HOME`, a caller without a `Runner.Worker` + ancestor, and `--negative-control` off a GitHub-hosted runner or in a + nested run, are refused (on macOS, `--negative-control` too). +- **L-01 to L-18, Linux only** (97 result lines; any other number fails + the suite). Each denial counts only next to its control, the same target + reached outside: + - **L-01:** host stream sockets, datagram `connect` and `sendto` refused + with `EACCES`, and an abstract socket unreachable (`ECONNREFUSED`, the + network namespace); the command carries exactly its profile; all four + targets reachable outside before and after; + - **L-02:** a host socket through a bind-mounted alias; **L-03:** in a + directory that cannot be listed (0311), of the host's and of another + network namespace, and a socket file without a listener + (`ECONNREFUSED`); **L-04:** a socket bound after the command started + (by timestamps); + - **L-05:** the gate's twin and the command's own stream and datagram + pairs work; + - **L-06:** a grandchild carries the profile, the command runs in the + gate's process, `change_profile`, `aa-exec -p unconfined` and `unshare + -U` are denied, no signal is ignored or blocked (read by `/bin/sh` + itself), `AF_VSOCK` is refused with `EACCES`; outside controls; + - **L-07:** a socket or a terminal as the caller's stdout is refused + before anything is made (controls: L-14's file, L-18's pipe); + - **L-08:** the gate run directly with a stray fd 5 and a socket on fd 1 + reports both and its missing confinement and refuses; **L-09:** + `--root-exec` closes inherited fds 5 and 6 and opens `/dev/null`; + - **L-10:** a host process that would pass a descriptor + (`SCM_RIGHTS`) cannot be reached; + - **L-11 and L-14:** the preflight's records, the profile listed while + the command runs, the gate's pass; afterwards no profile, control or + fence directory and no listener is left, the descriptors and + relays are logged, the relays end within a second and a normal end + calls no `kill-init`; + - **L-12:** the negative control is refused (fence reachable), only + `/usr/bin/true` could run, its profile is released; + - **L-13, L-13b, L-13c:** the release is refused while a process of the + suite outside carries the profile; the launcher then exits 125 and keeps + its control directory, from which the profile is released once the + carrier ended; two launches at once release only their own profiles + and leave no control or run directory; + - **L-15, L-15b, L-15c:** SIGTERM to `no-docker.sh` (143, pid 1 killed, + released after P1, nothing left); SIGKILL of the launch's `unshare` + while the command forks and tries a host socket (released after P1, or + left loaded and released afterwards; the socket never accepted); and + SIGTERM at 0 to 300 ms into the launch, each run ending in an allowed + outcome and no profile left; + - **L-17 and L-18:** 1 MiB of stdout and 64 KiB of stderr arrive + complete through a pipe; the command's last `::error::` line comes + before the resume token, which is the last stdout line; + - **L-16, last of all the cases that launch:** 10 launches are timed and + the guard job projected (184 s plus the suite's launches times the mean + launch time): above 540 s fails, compared exactly before any rounding + (the printed mean and projection are rounded up); above 360 s is a + warning. +- **T-02** also requires that the refusal comes from the probe's mechanism + check and that its writes and connects never ran. +- **T-36:** pure tests of the Linux rules (both platforms): the credential, + pid namespace, process, mount and descriptor rules, the gate's label, + descriptor and fence checks, `--root-exec`'s sanitation and the gating, + on fixtures only, with every call that would read `/proc`, open, close or + duplicate a descriptor, open a socket, run or exec a program, fork or set + a signal made to raise. The number of assertions per case is fixed in + the suite (135); fourteen mutated copies of the probe (among them the old + `NSpid` rule and a stacked label accepted) must each be rejected, and + checks that the probe calls the tested functions, in order, must pass. +- **T-38:** pure tests of `no-docker-aa.py` (both platforms): names, + labels, the preflight, the profile list, the sweeps, P0, P1 and the + state rules (with the namespace record's exact format), the template, + the negative template's reach, P2, `kill-init`'s order and the argument + validation, run against a fake system (114 assertions). The profile list + is read to its end (the kernel hands it out about a page per read), and + a task's label read that fills its buffer, which may have cut the label, + counts as unknown. Nineteen mutated copies (a single sweep, P1 or P2 + skipped, `kill-init` signalling before recording, the list read once, a + full label buffer accepted, and so on) must each be rejected. +- **T-39:** static rules (both platforms), each passing on the tracked + files and failing on a mutant fixture held in memory: no AppArmor reload, + replacement, removal or configuration write; `attach_disconnected` only + in the negative template; `sudo` in `no-docker.sh` only in its seven + reviewed commands, and `kill-init` only where `sudo` may still run; no + `sudo -E`; AppArmor only in the Linux arms; the templates exactly the + reviewed rules; no socket listing left; the negative control only where + it belongs; the pinned template hashes; a failed removal of the fence, + control or run directory is a cleanup failure. +- **T-40:** a pure harness (both platforms) of the Linux cleanup: the + `cleanup` function of `no-docker.sh` and the functions it calls, taken + from the script, run in a shell of their own with `rm`, `sudo` and + `chmod` replaced by stubs, on the suite's own directories. A failed + removal of the fence, control or run directory must end 125 with a + refusal naming that directory by its path, and leave exactly the + directories the design keeps (after a fence failure the control + directory too); a launch without one keeps the command's status and + leaves none. Four mutants of that propagation must each be caught. It + tests the shell logic only; L-14 and L-13c show on Linux that nothing is + left. +- **T-41:** a pure check (both platforms) of L-16's projection: from fixed + timings, exactly 540 s passes and 540.004 s or 540.04 s (which would + print as 540) fail, so no rounding can turn a projection above 540 s + into a pass; L-16 must use that computation. +- **T-42:** a pure check (both platforms) of the opt-in copy of the case + outputs: with `NO_DOCKER_TEST_EVIDENCE` set to an empty directory of mode + 0700, given by its canonical physical path (no component of it a + symbolic link; on macOS `/private/var/...`, not `/var/...`), the suite + copies its own outputs and the `--work` directories of its cases there + before removing them (regular files with safe names and a single link + only, with `MANIFEST.sha256` and `SKIPPED.txt`), so a CI job can export + them; a copy that cannot be completed fails the suite. Both trees are + walked by directory descriptors, each directory opened from its opened + parent without following a link, and files are read and written + relative to their opened directory, so a directory replaced by a link + while the copy runs is skipped, not read through. The sources are given + by their canonical physical paths as well (the suite makes its + directories physical when it creates them) and are never resolved: a + source with a link in its path is refused before anything is written. + T-42 replaces fixture directories by links to outside at the moments + that matter and checks that no byte from outside is kept; a destination + or a source with a link among its ancestors, or a source that is a link, + is refused. The full outputs of T-19, T-36, T-38 and T-39 are + written to `T-19.out`, `T-36.out`, `T-38.out` and `T-39.out` in the + suite's own directory before they are checked, so the copy keeps them, + and so are the streams of the launches whose output a case only filters + or discards: `t01.out`, `t02.out`, `t07.out` and `t07.err`, `t16.out` + with the socket records `t16-sockets` and `t16-classes`, `t17.out`, + `t17.err` and `t17c.out`, `t18-1.out` … `t18-7.err` and `t18.rc.out`, and + `l16.out`. Without the variable nothing changes. +- **T-43:** a pure check (both platforms) of `keep_stream`, through which + the launches of T-02, T-07, T-16, T-17 and T-18 keep the stream a case + reads (copied by `tee`, the launch still writing to a pipe): each mode + keeps exactly its stream and returns the command's own status, and a copy + that cannot be written ends the suite with 2 (a harness error) even after + the status the case expects; no other `tee` pipeline exists in the suite. +- **T-44:** a pure check (both platforms) of L-15c's result lines: at each + of the six moments a true verdict passes and a false one fails, under + the exact name of that moment. The earlier form, whose name held a + command substitution that set the status the result line took (every + moment but 300 ms reported failed and 300 ms passed, whatever the launch + did), is caught. +- **T-45:** a pure check (both platforms) of L-07's caller, with a stub in + place of `no-docker.sh`: the pty case gives it a live terminal on fd 1 + (the pseudo-terminal's master stays open in the caller, which drains it + and returns the launcher's status), the socketpair case a socket, and + neither any descriptor above 2. The earlier caller, whose exec closed the + master, gave a hung-up terminal, which Linux does not report as a + terminal: the launcher then refused only when it could not write its + output, after it had made its resources. It is caught. +- **T-46:** a pure check (both platforms) of how L-18 counts its 64 KiB + stderr line: byte for byte, with no pattern, counting only complete + lines (65535 `e` and their newline, from the start of the file or a + newline), so exactly one such line counts, lines one byte shorter, longer + or altered do not, and a file whose last line has no newline gives no + count at all. + The earlier count used `grep -c '^e\{65535\}$'`, whose repetition bound + GNU grep refuses ("Regular expression too big"): on Linux it gave no + number and L-18 could not pass. The case prints, as information, what + the platform's grep makes of that pattern. +- **T-47:** a pure check (both platforms) of T-07's verdict: the launch must + end 0, the filter of its kept stdout must end 0, and that stdout, without + the workflow command lines, must be exactly `refused reached`; the same + output with status 125, another output, none, or the right text from a + filter that ended 2 fail. T-07 takes the status of its launch from + `keep_stream` right after it; the earlier T-07 took `grep`'s status for + it (no `pipefail`), so a launch that printed the right text and then + ended 125 passed. That form is caught. +- **T-19:** the static CI order check, on both platforms. +- **T-20:** a static check that the code run inside the isolation has no + here-document or here-string, on both platforms. + +The macOS cases run only on a developer's Mac: there is no macOS CI job. + +#### Mutation testing + +- Run a mutated copy of `bin/devenv` or of a guard only through these + suites, on a macOS developer machine or in CI. On Linux outside CI the + suites refuse to run. +- Until a CI run on the GitHub-hosted runner has shown the Linux cases + L-01, L-02, L-03, L-04, L-12, L-15 and T-16 passing, use only payloads + that read or print, on both platforms. +- Never use a destructive Docker command (prune, forced removal, `down -v`, + `--rmi`), not even in isolation. +- `tests/devenv/integration.sh` and the `tests/app/` suites use the real + Docker daemon on purpose, so never run them on a mutated copy. + `integration.sh` refuses a `DEVENV` that differs from `bin/devenv`, but no + suite can tell an edited checkout from a mutated one. +- BE-18 mutates a copy of the backend inside the api test container, which + has no Docker socket. + Application suites (`tests/app/`), each in its own Compose projects: - `scan.sh` (SC-xx): Trivy scans of the runtime images (production-like api diff --git a/scripts/ci/fdnorm.py b/scripts/ci/fdnorm.py new file mode 100644 index 0000000..30dac86 --- /dev/null +++ b/scripts/ci/fdnorm.py @@ -0,0 +1,246 @@ +"""Start an isolated CI suite with no descriptor above 2. + +Usage: python3 -I scripts/ci/fdnorm.py -- COMMAND + +Steps of GitHub-hosted jobs were seen to inherit open pipes above fd 2 (two +pipes on fds 142 and 145 in run 41636839721). Which process opened them is +not known. The isolation launcher refuses any descriptor it did not open +itself (ADR-023), so each isolated step of .github/workflows/ci.yml starts +through this program. It changes nothing in the launcher or in what the +launcher, --root-exec or the gate check, and it is not an isolation: COMMAND +still has to isolate itself (scripts/ci/tests/ci-isolation-order.py). + +In this process, which then becomes COMMAND (same pid): + 0. on Linux, its start time (field 22 of /proc/self/stat) must be read + and be a positive decimal number, else refused (off Linux, where + this program runs only for local tests, it is reported as "-"); + 1. fds 0, 1 and 2 must be open and not close-on-exec; every open + descriptor is listed from /proc/self/fd (macOS, for local tests: + /dev/fd) and examined with fstat and fcntl. A listing that fails, a + name that is not a decimal number, more than MAX_FDS open + descriptors, an fstat or fcntl error other than the listing's own + (already closed) descriptor answering EBADF, or more than one such + descriptor: refused; + 2. the hard RLIMIT_NOFILE must be a finite integer from 3 to MAX_LIMIT, + and above every open descriptor: else refused (closerange below + would leave a descriptor at or above it open); + 3. os.closerange(3, that limit): every descriptor above 2, no list of + numbers kept; + 4. the descriptors are listed and examined again as in 1: only 0, 1 and + 2 may be open, each with the file type, device, inode, device + number, status flags and descriptor flags it had in 1; else refused; + 5. the two listings are written to fd 2 (lines "ci-fd: ..."; at most + MAX_OUT bytes, else refused, never cut), then SIGPIPE and SIGXFSZ, + which the Python runtime ignores, are set back to their default (as + subprocess does for its children); + 6. COMMAND is executed (os.execvp). Nothing is opened after 4. + +A refusal is reported on fd 2 and ends with status 2; COMMAND is not +run. If COMMAND cannot be executed the status is 127. Reported per +descriptor: number, file type, device, inode, device number of a device, +access mode, O_APPEND, O_NONBLOCK and close-on-exec; for the process: its +pid, parent pid, uids and start time (Linux). Nothing else: no path a +descriptor names, no data, no environment, no argument, no other process. +A descriptor the caller marked close-on-exec was closed when Python +started, so it is neither listed nor reported as closed here. Which +process opened an inherited descriptor is not shown and not claimed. +""" + +import errno +import fcntl +import os +import resource +import signal +import stat +import sys +import warnings # noqa: F401 os.execvp imports it while searching PATH: loaded before step 1 + +PREFIX = b"ci-fd: " +MAX_FDS = 1024 +MAX_OUT = 1 << 18 +MAX_LIMIT = 1 << 30 +LINUX = os.path.isdir("/proc/self/fd") +FDDIR = "/proc/self/fd" if LINUX else "/dev/fd" +STAT = "/proc/self/stat" if LINUX else None # off Linux: no start time (local tests) +TYPES = ((stat.S_ISFIFO, "fifo"), (stat.S_ISSOCK, "socket"), (stat.S_ISREG, "file"), (stat.S_ISCHR, "char"), + (stat.S_ISDIR, "dir"), (stat.S_ISBLK, "block"), (stat.S_ISLNK, "link")) +ACCESS = {os.O_RDONLY: "r", os.O_WRONLY: "w", os.O_RDWR: "rw"} + + +class Refused(Exception): + pass + + +class System: + """The calls this program makes; the tests replace them.""" + + listdir = staticmethod(os.listdir) + fstat = staticmethod(os.fstat) + closerange = staticmethod(os.closerange) + write = staticmethod(os.write) + execvp = staticmethod(os.execvp) + signal = staticmethod(signal.signal) + + @staticmethod + def flags(fd): + return fcntl.fcntl(fd, fcntl.F_GETFL), fcntl.fcntl(fd, fcntl.F_GETFD) + + @staticmethod + def hard_limit(): + return resource.getrlimit(resource.RLIMIT_NOFILE)[1] + + @staticmethod + def process(): + """This process's identity; on Linux its start time is required.""" + start = "-" + if STAT is not None: + try: + with open(STAT, "rb") as f: + data = f.read(4096) + except OSError as e: + raise Refused("cannot read %s (%s)" % (STAT, errname(e))) + fields = data.rsplit(b")", 1)[1].split() if b")" in data else [] + field = fields[19] if len(fields) > 19 else b"" # field 22: the fields after the name start at 3 + if not (field.isdigit() and field[:1] != b"0" and len(field) <= 20): + raise Refused("no start time in %s (field 22 is not a positive decimal number)" % STAT) + start = field.decode("ascii") + r, e = os.getuid(), os.geteuid() + return "pid %d ppid %d uid %d euid %d start %s" % (os.getpid(), os.getppid(), r, e, start) + + +def errname(e): + return errno.errorcode.get(e.errno, str(e.errno)) + + +def describe(st, fl, fdfl): + """The reported identity of one descriptor (a tuple, compared in 4).""" + kind = next((name for test, name in TYPES if test(st.st_mode)), "other") + rdev = st.st_rdev if kind in ("char", "block") else None + return (kind, st.st_dev, st.st_ino, rdev, fl, fdfl) + + +def row(fd, ident): + kind, dev, ino, rdev, fl, fdfl = ident + extra = [] if rdev is None else ["rdev %d" % rdev] + flags = [n for n, b in (("append", os.O_APPEND), ("nonblock", os.O_NONBLOCK)) if fl & b] + return "fd %d: %s, dev %d, ino %d%s, %s%s, cloexec %d" % ( + fd, kind, dev, ino, "".join(", " + x for x in extra), ACCESS.get(fl & os.O_ACCMODE, "?"), + "".join(", " + x for x in flags), 1 if fdfl & fcntl.FD_CLOEXEC else 0) + + +def listing(system): + """{fd: identity} of every open descriptor of this process.""" + try: + names = system.listdir(FDDIR) + except OSError as e: + raise Refused("cannot list %s (%s)" % (FDDIR, errname(e))) + if len(names) > MAX_FDS + 1: + raise Refused("%d descriptors listed, more than %d" % (len(names), MAX_FDS)) + found, gone = {}, [] + for name in names: + if not (isinstance(name, str) and name.isascii() and name.isdigit()): + raise Refused("%r in %s is not a descriptor number" % (name, FDDIR)) + fd = int(name) + try: + st = system.fstat(fd) + except OSError as e: + if e.errno == errno.EBADF and fd > 2: + gone.append(fd) + continue + raise Refused("fd %d cannot be examined (fstat: %s)" % (fd, errname(e))) + try: + fl, fdfl = system.flags(fd) + except OSError as e: + raise Refused("fd %d cannot be examined (fcntl: %s)" % (fd, errname(e))) + found[fd] = describe(st, fl, fdfl) + if len(gone) > 1: + raise Refused("fds %s were listed but are closed (only the listing's own may be)" % sorted(gone)) + if len(found) > MAX_FDS: + raise Refused("%d descriptors open, more than %d" % (len(found), MAX_FDS)) + return found + + +def write_all(system, data): + while data: + n = system.write(2, data) + if n <= 0: + raise OSError(errno.EIO, "short write") + data = data[n:] + + +def report(system, lines): + """Write LINES to fd 2, all of them or (more than MAX_OUT bytes) none.""" + text = b"".join(PREFIX + line.encode("ascii", "backslashreplace") + b"\n" for line in lines) + if len(text) > MAX_OUT: + raise Refused("the report would be %d bytes, more than %d" % (len(text), MAX_OUT)) + write_all(system, text) + + +def normalize(argv, lines, system=System): + """Steps 1 to 5, the report collected in LINES; returns COMMAND. + Raises Refused, or OSError if the report cannot be written.""" + if len(argv) < 3 or argv[1] != "--": + raise Refused("usage: fdnorm.py -- COMMAND [ARG...]") + command = argv[2:] + lines.append("before: " + system.process()) + before = listing(system) + lines += [row(fd, before[fd]) for fd in sorted(before)] + for fd in (0, 1, 2): + if fd not in before: + raise Refused("fd %d is not open" % fd) + if before[fd][5] & fcntl.FD_CLOEXEC: + raise Refused("fd %d is close-on-exec" % fd) + limit = system.hard_limit() + if type(limit) is not int or limit == resource.RLIM_INFINITY or not 3 <= limit <= MAX_LIMIT: + raise Refused("the hard RLIMIT_NOFILE (%r) is not a finite number from 3 to %d" % (limit, MAX_LIMIT)) + high = sorted(fd for fd in before if fd >= limit) + if high: + raise Refused("fds %s are at or above the hard RLIMIT_NOFILE %d" % (high, limit)) + system.closerange(3, limit) + after = listing(system) + left = sorted(fd for fd in after if fd > 2) + if left: + raise Refused("fds %s are still open" % left) + for fd in (0, 1, 2): + if after.get(fd) != before[fd]: + raise Refused("fd %d changed" % fd) + closed = sorted(fd for fd in before if fd > 2) + lines += ["after: " + row(fd, after[fd]) for fd in (0, 1, 2)] + lines.append("closed %s (up to the hard limit %d); open now: 0 1 2, as before; executing the command" + % (" ".join(map(str, closed)) or "none", limit)) + report(system, lines) + system.signal(signal.SIGPIPE, signal.SIG_DFL) + system.signal(signal.SIGXFSZ, signal.SIG_DFL) + return command + + +def main(argv, system=System): + lines = [] + try: + command = normalize(argv, lines, system) + except Refused as e: + refusal = "refused: %s; the command was not run" % e + try: + report(system, lines + [refusal]) + except (Refused, OSError): + try: + report(system, [refusal]) + except (Refused, OSError): + pass + return 2 + except OSError: + return 2 + try: + system.execvp(command[0], command) + except OSError as e: + system.signal(signal.SIGPIPE, signal.SIG_IGN) + try: + report(system, ["the command could not be executed (%s)" % errname(e)]) + except (Refused, OSError): + pass + return 127 + return 127 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/ci/no-docker-aa.py b/scripts/ci/no-docker-aa.py new file mode 100644 index 0000000..873bac5 --- /dev/null +++ b/scripts/ci/no-docker-aa.py @@ -0,0 +1,981 @@ +"""The AppArmor profile of one Linux launch of scripts/ci/no-docker.sh. + +Run as root through sudo by no-docker.sh only, in these argv shapes +(sudo -n timeout N /usr/bin/python3 -I ...): + + preflight refuse unless AppArmor can confine the launch: + enabled, global mode enforce, the af_unix network + class, apparmor_parser and aa-exec present; print + what else matters for the record; + load NAME CTLDIR load the pinned production template as NAME + (add only, never replace) and record its kernel + hash in CTLDIR/hash; + load-negative-control NEGNAME CTLDIR + the same with the pinned negative-control + template (attach_disconnected), only for NEGNAME; + kill-init LAUNCHERPID CTLDIR + find pid 1 of the launch's pid namespace from the + launcher's process tree, record its namespace in + CTLDIR/ns if that is missing, then SIGKILL it + through a pidfd; nothing else is ever signalled; + release NAME|NEGNAME CTLDIR + remove the profile only when no task can carry it + (P0, P1, P2 and two sweeps); otherwise leave it + loaded and fail; + diag NAME|NEGNAME print what is known about the profile. + +Inputs are the argv, validated, and the two values sudo sets itself, +SUDO_UID and SUDO_USER; no other environment value is read. Exit 0 on +success, 1 when refused or when the profile is left loaded, 2 on a usage +error. Messages go to stderr. + +Release (docs/dev-environment.md, "Profile lifecycle"): + P0 CTLDIR/ns is a root-owned regular file of mode 0444 in the format + "pid: mnt:", and its pid namespace is not ours; + P1 no task is in that pid namespace, except its init as a single-threaded + zombie: when a pid namespace's init exits, the kernel kills every other + task of the namespace and lets none be created before init becomes a + zombie (zap_pid_ns_processes); + P2 NAME is listed exactly once, in enforce mode, with the kernel hash + recorded at load time; + S two consecutive complete sweeps of every task's + attr/apparmor/current find no label containing NAME; + R NAME is removed and is no longer listed. +State "loading" or "not-forked" with no CTLDIR/ns means no namespace was +ever made: then NAME is removed after P2 and the sweeps, or nothing is done +when it was never loaded. Any error other than a task that ended is unknown, +and unknown means the profile stays loaded. +""" + +import errno +import hashlib +import os +import pwd +import re +import signal +import stat +import subprocess +import sys +import time + +SECURITYFS = "/sys/kernel/security/apparmor" +PROFILES = SECURITYFS + "/profiles" +REMOVE = SECURITYFS + "/.remove" +POLICY = SECURITYFS + "/policy/profiles" +FEATURES = SECURITYFS + "/features" +PARAMETERS = "/sys/module/apparmor/parameters" +PARSER = "/usr/sbin/apparmor_parser" +AA_EXEC = "/usr/bin/aa-exec" +HERE = os.path.dirname(os.path.abspath(__file__)) +TEMPLATES = { + "production": (os.path.join(HERE, "no-docker", "apparmor.profile"), + "035a67af7a79fdfd2e9530955f7fba3dfede64bdb5af04651967a8b1d0f2319d"), + "negative": (os.path.join(HERE, "tests", "no-docker", "apparmor-negative.profile"), + "2375d20c5202891297e686ced0976899e36601d9824d475f6f34c204b28ea46e"), +} +NAME_RE = re.compile(r"flowcrm-nd-[0-9a-f]{32}") +NEGNAME_RE = re.compile(r"flowcrm-nd-neg-[0-9a-f]{32}") +CTL_BASE_RE = re.compile(r"no-docker-ctl\.[A-Za-z0-9]{6,}") +USER_RE = re.compile(r"[a-z_][a-z0-9_-]{0,31}") +NS_RE = re.compile(r"pid:([1-9][0-9]{0,19}) mnt:([1-9][0-9]{0,19}|-)\n") +STATE_RE = re.compile(r"(loading|not-forked|entering|entered spid=[1-9][0-9]{0,9})\n") +HASH_RE = re.compile(r"sha256 ([0-9a-f]{64}|none)\n") +# The argv of the sudo that starts the namespaces (no-docker.sh), up to the +# init script. +UNSHARE_ARGV = ["unshare", "--net", "--mount", "--pid", "--fork", "--kill-child", "--mount-proc", "--ipc", + "--propagation", "private", "--", "/bin/sh", "-c"] +SUDO_ARGV = ["sudo", "-n"] + UNSHARE_ARGV +KILL_INIT_DEADLINE = 8.0 +SWEEPS = 2 +# A task's label is read in one read of LABEL_MAX bytes (a read that fills it +# may have cut the label); the profile list is read to its end, at most +# LIST_MAX bytes. +LABEL_MAX = 1 << 16 +LIST_MAX = 1 << 24 + + +class Refused(Exception): + """The operation cannot be done safely; the message says why.""" + + +def say(text): + print(f"no-docker-aa: {text}", file=sys.stderr, flush=True) + + +def gone(e): + """True for the errors of a task that ended meanwhile.""" + return isinstance(e, OSError) and e.errno in (errno.ENOENT, errno.ESRCH) + + +# --- pure decisions ------------------------------------------------------------ + +def name_ok(op, name): + """The profile names each operation accepts.""" + production = NAME_RE.fullmatch(name) is not None + negative = NEGNAME_RE.fullmatch(name) is not None + if op == "load": + return production + if op == "load-negative-control": + return negative + if op in ("release", "diag"): + return production or negative + return False + + +def template_for(op, name): + """(kind, path, pinned sha256) of the template an operation loads.""" + if op == "load" and name_ok("load", name): + return ("production",) + TEMPLATES["production"] + if op == "load-negative-control" and name_ok("load-negative-control", name): + return ("negative",) + TEMPLATES["negative"] + raise Refused(f"{op} does not load {name}") + + +def render(data, kind, sha, name): + """The profile text for name, from the template bytes data; refused + unless the bytes are the pinned ones and the result is plain.""" + if hashlib.sha256(data).hexdigest() != sha: + raise Refused("the template is not the reviewed one (sha256 differs)") + text = data.decode("ascii").replace("@NAME@", name) + problems = [] + if "@" in text: + problems.append("a placeholder is left") + for token in ("change_profile", "complain", "userns", "include"): + if token in text: + problems.append(token) + if re.search(r"\b[pPcCuU]i?x\b|\bix\b", text): + problems.append("an exec transition") + if "attach_disconnected" in text and kind != "negative": + problems.append("attach_disconnected") + if kind == "negative" and "flags=(enforce,attach_disconnected)" not in text: + problems.append("the negative template without attach_disconnected") + if not re.search(r"^profile " + re.escape(name) + r" flags=\(", text, re.M): + problems.append("the profile is not named " + name) + if problems: + raise Refused("the rendered profile is not plain: " + ", ".join(problems)) + return text + + +def parse_label(line): + """(name, mode) of a label or profile list line "NAME (mode)"; (name, + None) for a label without a mode (unconfined); None if empty.""" + line = line.rstrip("\n") + if not line.strip(): + return None + m = re.fullmatch(r"(.+) \(([a-z_]+)\)", line) + if m: + return m.group(1), m.group(2) + return line, None + + +def listed_modes(text, name): + """The modes NAME is listed with in the profile list text.""" + modes = [] + for line in text.splitlines(): + label = parse_label(line) + if label and label[0] == name: + modes.append(label[1]) + return modes + + +def preflight_findings(o): + """(passed, text) for each gating check of the preflight observation o; + the sysctl values are recorded, not gating.""" + return [ + (o.get("enabled") == "Y", f"AppArmor is enabled ({o.get('enabled')})"), + (o.get("mode") == "enforce", f"the global mode is enforce ({o.get('mode')})"), + (o.get("af_unix") is True, f"the kernel mediates af_unix (features/network/af_unix: {o.get('af_unix')})"), + (o.get("parser") is True, f"{PARSER} is present ({o.get('parser')})"), + (o.get("aa_exec") is True, f"{AA_EXEC} is present ({o.get('aa_exec')})"), + ] + + +def af_unix_feature(kind, text): + """True if the kernel's features/network/af_unix entry says AppArmor + mediates Unix sockets: a directory, or a file reading "yes" (as on the + Ubuntu runner kernels); False if missing or anything else.""" + if kind == "dir": + return True + return kind == "file" and text.strip() == "yes" + + +def parse_state(text): + """The launch state CTLDIR/state names; "unknown" for anything else.""" + if text is None or STATE_RE.fullmatch(text) is None: + return "unknown" + return text.rstrip("\n").split(" ")[0] + + +def parse_ns(text): + """(pid namespace inode, mount namespace inode or None) of CTLDIR/ns; + ValueError unless it is exactly "pid: mnt:" and a newline.""" + m = NS_RE.fullmatch(text) + if not m: + raise ValueError(f"not a namespace record: {text[:80]!r}") + pid, mnt = int(m.group(1)), m.group(2) + if pid >= 1 << 64 or (mnt != "-" and int(mnt) >= 1 << 64): + raise ValueError("a namespace inode out of range") + return pid, None if mnt == "-" else int(mnt) + + +def ns_record(pid_ino, mnt_ino): + return f"pid:{pid_ino} mnt:{'-' if mnt_ino is None else mnt_ino}\n" + + +def release_plan(state, ns_exists): + """"no-namespace" when no namespace can have been made (the state says + no fork happened and no namespace was recorded), else "namespace".""" + if state in ("loading", "not-forked") and not ns_exists: + return "no-namespace" + return "namespace" + + +def p0(rec, host_ino): + """None if the namespace record rec ({"uid", "mode", "regular", "text"} or + None if missing) is valid and names another pid namespace, else why not.""" + if rec is None: + return "no namespace record (CTLDIR/ns)" + if not rec["regular"] or rec["uid"] != 0 or rec["mode"] != 0o444: + return f"the namespace record is not a root-owned file of mode 0444 (uid {rec['uid']}, mode {rec['mode']:o})" + try: + pid_ino, _ = parse_ns(rec["text"]) + except ValueError as e: + return str(e) + if pid_ino == host_ino: + return "the namespace record names the host's pid namespace" + return None + + +def p1(tasks, pid_ino): + """None if no task is in the pid namespace pid_ino except its init as a + single-threaded zombie, else why not. tasks: (pid, its pid namespace + inode or an OSError, its status or an OSError).""" + found = [] + for pid, ino, st in tasks: + if isinstance(ino, OSError): + if not gone(ino): + return f"task {pid}: its pid namespace cannot be read ({ino.strerror})" + continue + if ino != pid_ino: + continue + if isinstance(st, OSError): + if gone(st): + continue + return f"task {pid} of the namespace cannot be read ({st.strerror})" + nspid = st.get("NSpid", "").split() + init = (len(nspid) >= 2 and nspid[-1] == "1" and st.get("Name") == "sh" and st.get("Threads") == "1" + and st.get("State", "").startswith("Z")) + if not init: + found.append(f"{pid}:{st.get('Name', '?')}:{st.get('State', '?')[:1]}") + if found: + return "tasks still in the namespace: " + " ".join(found) + return None + + +def p2(modes, hash_now, hash_load): + """None if NAME is listed once in enforce mode with its load-time hash + (hash_load: "none" if the kernel exported none at load), else why not.""" + if modes != ["enforce"]: + return f"the profile vanished or was replaced during the run (listed as {modes or 'nothing'})" + if hash_load is None: + return "no load-time hash was recorded" + if (hash_now or "none") != hash_load: + return f"the profile's kernel hash changed ({hash_load} at load, {hash_now or 'none'} now)" + return None + + +def sweep_result(entries, name): + """(carriers, unknown) of one sweep. entries: (path, the label or the + OSError reading it). Any label containing NAME counts (stacked and + prefixed renderings, and longer names that contain it).""" + carriers, unknown = [], [] + for path, label in entries: + if isinstance(label, OSError): + if not gone(label): + unknown.append(f"{path} ({label.strerror})") + elif name in label: + carriers.append(path) + return carriers, unknown + + +def sweeps_clear(sweep, name, count=SWEEPS): + """None if count consecutive complete sweeps find no carrier, else why.""" + for i in range(count): + carriers, unknown = sweep_result(sweep(), name) + if unknown: + return f"sweep {i + 1} could not read: {' '.join(unknown[:5])}" + if carriers: + return f"sweep {i + 1} found {len(carriers)} tasks carrying the profile: {' '.join(carriers[:5])}" + return None + + +def read_all(read, limit): + """The bytes of a file read with read(n) until it returns none (its + end): the kernel hands the profile list out about a page per read (a + seq_file), so one read may hold only its first part. OSError if there + are more than limit bytes.""" + data = b"" + while True: + chunk = read(1 << 16) + if not chunk: + return data + data += chunk + if len(data) > limit: + raise OSError(errno.EFBIG, f"more than {limit} bytes") + + +def label_from(data, limit): + """The text of a task's label from one read of attr/apparmor/current + with a buffer of limit bytes (the kernel gives the whole label in one + read, cut to the buffer). OSError if the read filled the buffer: the + label may have been cut, and a profile named after the cut would not be + seen.""" + if len(data) >= limit: + raise OSError(errno.EOVERFLOW, f"a label of {limit} bytes or more may have been cut") + return data.decode("utf-8", "replace") + + +def parse_stat(text): + """{"comm", "state", "ppid", "start"} of a /proc//stat line.""" + head, _, rest = text.rpartition(")") + f = rest.split() + return {"comm": head.partition("(")[2], "state": f[0], "ppid": int(f[1]), "start": int(f[19])} + + +def ctldir_findings(path, info, real, sudo_uid, sudo_user, user_uid): + """Why the CTLDIR argument is refused ([] if valid). info: lstat of path + ({"dir", "link", "uid", "mode"}) or None; real: its realpath; user_uid: + the uid of sudo_user in the user database (None if unknown).""" + f = [] + if not path.startswith("/") or os.path.normpath(path) != path: + f.append("not an absolute canonical path") + if real != path: + f.append("a symbolic link in its path") + if info is None or not info["dir"] or info["link"]: + f.append("not a directory") + elif info["mode"] != 0o700 or info["uid"] != sudo_uid: + f.append(f"not mode 0700 owned by uid {sudo_uid}") + if not (sudo_user and USER_RE.fullmatch(sudo_user)) or user_uid is None or user_uid != sudo_uid: + f.append("SUDO_USER and SUDO_UID do not name one user") + elif os.path.dirname(path) != f"/home/{sudo_user}/work/_temp": + f.append(f"not in /home/{sudo_user}/work/_temp (the runner's temporary directory)") + if not CTL_BASE_RE.fullmatch(os.path.basename(path)): + f.append("its name is not no-docker-ctl.XXXXXX") + return f + + +def launcher_findings(launcher, procs, me, sudo_uid): + """Why LAUNCHERPID is refused ([] if valid): it must be a process of + SUDO_UID, and walking up from this helper (helper, timeout, sudo, maybe + sudo's monitor; at most 5 levels) must reach a sudo whose parent it is. + procs: {pid: {"ppid", "comm", "ruid"}}.""" + p = procs.get(launcher) + if p is None: + return ["the launcher does not exist"] + if p["ruid"] != sudo_uid: + return [f"the launcher runs as uid {p['ruid']}, not {sudo_uid}"] + cur = me + for _ in range(5): + info = procs.get(cur) + if info is None: + break + cur = info["ppid"] + up = procs.get(cur) + if up and up["comm"] == "sudo" and up["ppid"] == launcher: + return [] + return ["this helper was not started by a sudo of the launcher"] + + +def find_init(procs, launcher, host_ino, recorded): + """("found", pid), ("none", why) or ("ambiguous", why): pid 1 of the + launch's namespace, found from the launcher's process tree. procs: {pid: + {"ppid", "comm", "argv", "start", "nspid", "pidns"}}; recorded: the + pid namespace inode of CTLDIR/ns or None.""" + children = {} + for pid, p in procs.items(): + children.setdefault(p["ppid"], []).append(pid) + sudos = [c for c in children.get(launcher, []) if procs[c]["comm"] == "sudo" + and procs[c]["argv"][:len(SUDO_ARGV)] == SUDO_ARGV] + if not sudos: + return "none", "no sudo of the launcher starts the namespaces" + if len(sudos) > 1: + return "ambiguous", f"{len(sudos)} sudo processes of the launcher start namespaces" + level, unshares = [sudos[0]], [] + for _ in range(3): + level = [c for p in level for c in children.get(p, [])] + # Only an unshare in the host's pid namespace: its forked child is + # in the new one (and so is anything the command could rename). + unshares += [c for c in level if procs[c]["comm"] == "unshare" and procs[c]["pidns"] == host_ino + and procs[c]["argv"][:len(UNSHARE_ARGV)] == UNSHARE_ARGV] + if not unshares: + return "none", "the sudo has not started unshare" + if len(unshares) > 1: + return "ambiguous", f"{len(unshares)} unshare processes under the sudo" + kids = children.get(unshares[0], []) + if not kids: + return "none", "unshare has not started pid 1" + if len(kids) > 1: + return "ambiguous", f"unshare has {len(kids)} children" + init = procs[kids[0]] + if init["pidns"] is None or init["pidns"] == host_ino: + return "none", "unshare's child is not in a new pid namespace yet" + if not init["nspid"] or init["nspid"][-1] != "1": + return "none", f"unshare's child is not pid 1 of its namespace (NSpid {' '.join(init['nspid'])})" + if recorded is not None and init["pidns"] != recorded: + return "ambiguous", "unshare's child is not in the recorded pid namespace" + return "found", kids[0] + + +# --- what the kernel and the files show (as root) -------------------------------- + +def read_text(path, limit=1 << 20): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + try: + return os.read(fd, limit).decode("utf-8", "replace") + finally: + os.close(fd) + + +def proc_pids(): + return [int(d) for d in os.listdir("/proc") if d.isdigit()] + + +def ns_inode(pid, kind): + """The inode of /proc//ns/ ("pid:[N]").""" + link = os.readlink(f"/proc/{pid}/ns/{kind}") + m = re.fullmatch(kind + r":\[([0-9]+)\]", link) + if not m: + raise OSError(f"unexpected namespace link {link!r}") + return int(m.group(1)) + + +def proc_status(pid): + st = {} + for line in read_text(f"/proc/{pid}/status").splitlines(): + k, _, v = line.partition(":") + st[k] = v.strip() + return st + + +def process_table(with_argv=True): + """{pid: {"ppid", "comm", "argv", "start", "nspid", "pidns", "ruid"}} of + every process that did not end while being read; any other error + raises.""" + procs = {} + for pid in proc_pids(): + try: + s = parse_stat(read_text(f"/proc/{pid}/stat")) + st = proc_status(pid) + argv = read_text(f"/proc/{pid}/cmdline").split("\0")[:-1] if with_argv else [] + try: + pidns = ns_inode(pid, "pid") + except OSError as e: + if not gone(e): + raise + pidns = None + except OSError as e: + if gone(e): + continue + raise + procs[pid] = {"ppid": s["ppid"], "comm": s["comm"], "start": s["start"], "argv": argv, + "nspid": st.get("NSpid", "").split(), "pidns": pidns, + "ruid": int(st.get("Uid", "-1").split()[0])} + return procs + + +def still_there(e, path): + """e, or an error that does not count as "ended" when path (the task's + /proc directory) still exists: only a task that is gone may be skipped.""" + if gone(e) and os.path.exists(path): + return OSError(errno.EIO, f"unreadable although {path} exists") + return e + + +def pidns_tasks(): + """(pid, pid namespace inode or OSError, status or OSError) of every + process, for P1.""" + tasks = [] + for pid in proc_pids(): + try: + ino = ns_inode(pid, "pid") + except OSError as e: + tasks.append((pid, still_there(e, f"/proc/{pid}"), None)) + continue + try: + st = proc_status(pid) + except OSError as e: + st = still_there(e, f"/proc/{pid}") + tasks.append((pid, ino, st)) + return tasks + + +def read_label(path): + """A task's label, read whole in one read (label_from).""" + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + try: + return label_from(os.read(fd, LABEL_MAX), LABEL_MAX) + finally: + os.close(fd) + + +def label_sweep(): + """(path, label or OSError) of every task's attr/apparmor/current. A + kernel without that file would make every task look ended: the sweep + refuses unless it can read its own.""" + read_label("/proc/self/attr/apparmor/current") + entries = [] + for pid in proc_pids(): + try: + tids = [d for d in os.listdir(f"/proc/{pid}/task") if d.isdigit()] + except OSError as e: + entries.append((f"/proc/{pid}/task", still_there(e, f"/proc/{pid}"))) + continue + for tid in tids: + path = f"/proc/{pid}/task/{tid}/attr/apparmor/current" + try: + entries.append((path, read_label(path))) + except OSError as e: + entries.append((path, still_there(e, f"/proc/{pid}/task/{tid}"))) + return entries + + +def profile_list(): + """The kernel's list of loaded profiles, read to its end (read_all).""" + fd = os.open(PROFILES, os.O_RDONLY | os.O_NOFOLLOW) + try: + return read_all(lambda n: os.read(fd, n), LIST_MAX).decode("utf-8", "replace") + finally: + os.close(fd) + + +def kernel_hash(name): + """The kernel's sha256 of the loaded profile name, or None if it exports + none.""" + try: + dirs = os.listdir(POLICY) + except FileNotFoundError: + return None + found = [] + for d in dirs: + try: + if read_text(f"{POLICY}/{d}/name", 4096).strip() != name: + continue + except FileNotFoundError: + continue + try: + found.append(read_text(f"{POLICY}/{d}/sha256", 4096).strip() or None) + except FileNotFoundError: + found.append(None) + if len(found) > 1: + raise OSError(f"{name} appears {len(found)} times in the policy tree") + return found[0] if found else None + + +class CtlDir: + """The launch's control directory, opened once (no symbolic link + followed) and used relative to that descriptor.""" + + def __init__(self, path): + self.path = path + self.fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + + def lstat(self, name): + try: + return os.lstat(name, dir_fd=self.fd) + except FileNotFoundError: + return None + + def read(self, name, limit=4096): + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=self.fd) + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + raise OSError(errno.EINVAL, f"{name} is not a regular file") + return os.read(fd, limit).decode("utf-8", "replace") + finally: + os.close(fd) + + def record(self, name): + """{"uid", "mode", "regular", "text"} of a file here, None if missing.""" + st = self.lstat(name) + if st is None: + return None + return {"uid": st.st_uid, "mode": stat.S_IMODE(st.st_mode), "regular": stat.S_ISREG(st.st_mode), + "text": self.read(name) if stat.S_ISREG(st.st_mode) else ""} + + def write_new(self, tmp, final, text, replace): + """Write text to tmp (created exclusively, root, 0444, synced), then + rename it over final (replace) or link it to final (no replace: False + if final exists).""" + if self.lstat(tmp) is not None: + os.unlink(tmp, dir_fd=self.fd) + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o444, dir_fd=self.fd) + try: + os.fchmod(fd, 0o444) + os.write(fd, text.encode()) + os.fsync(fd) + finally: + os.close(fd) + try: + if replace: + os.rename(tmp, final, src_dir_fd=self.fd, dst_dir_fd=self.fd) + return True + try: + os.link(tmp, final, src_dir_fd=self.fd, dst_dir_fd=self.fd) + return True + except FileExistsError: + return False + finally: + if self.lstat(tmp) is not None: + os.unlink(tmp, dir_fd=self.fd) + os.fsync(self.fd) + + +def validated_ctldir(path): + sudo_uid, sudo_user = sudo_identity() + try: + st = os.lstat(path) + info = {"dir": stat.S_ISDIR(st.st_mode), "link": stat.S_ISLNK(st.st_mode), "uid": st.st_uid, + "mode": stat.S_IMODE(st.st_mode)} + except FileNotFoundError: + info = None + found = ctldir_findings(path, info, os.path.realpath(path), sudo_uid, sudo_user, user_uid(sudo_user)) + if found: + raise Refused(f"CTLDIR {path} refused: {'; '.join(found)}") + ctl = CtlDir(path) + st = os.fstat(ctl.fd) + if st.st_uid != sudo_uid or stat.S_IMODE(st.st_mode) != 0o700: + raise Refused(f"CTLDIR {path} changed while it was opened") + return ctl + + +def sudo_identity(): + uid = os.environ.get("SUDO_UID", "") + if not uid.isdigit() or int(uid) == 0: + raise Refused("SUDO_UID is not the uid of an unprivileged caller") + return int(uid), os.environ.get("SUDO_USER", "") + + +def user_uid(user): + try: + return pwd.getpwnam(user).pw_uid if user and USER_RE.fullmatch(user) else None + except KeyError: + return None + + +# --- the operations -------------------------------------------------------------- + +class System: + """The I/O of the operations, as root. The unit tests replace it.""" + + def preflight_observation(self): + def param(name): + try: + return read_text(f"{PARAMETERS}/{name}", 4096).strip() + except OSError: + return None + + def present(path): + try: + st = os.stat(path) + return stat.S_ISREG(st.st_mode) and bool(st.st_mode & 0o111) + except FileNotFoundError: + return False + except OSError: + return None + + path = f"{FEATURES}/network/af_unix" + try: + st = os.stat(path) + kind = "dir" if stat.S_ISDIR(st.st_mode) else "file" if stat.S_ISREG(st.st_mode) else "other" + af_unix = af_unix_feature(kind, read_text(path, 4096) if kind == "file" else "") + except FileNotFoundError: + af_unix = False + except OSError: + af_unix = None + return {"enabled": param("enabled"), "mode": param("mode"), "af_unix": af_unix, + "parser": present(PARSER), "aa_exec": present(AA_EXEC)} + + def preflight_record(self): + """What the launch log records, never gating.""" + lines = [] + + def read(label, path): + try: + lines.append(f"{label} {read_text(path, 4096).strip() or '(empty)'}") + except OSError as e: + lines.append(f"{label} unreadable ({e.strerror})") + + def run(label, argv): + try: + r = subprocess.run(argv, capture_output=True, text=True, timeout=5, stdin=subprocess.DEVNULL) + out = (r.stdout + r.stderr).strip().splitlines() + return out + except (OSError, subprocess.SubprocessError) as e: + lines.append(f"{label} unavailable ({e})") + return [] + + out = run("parser", [PARSER, "--version"]) + if out: + lines.append(f"parser {out[0]}") + try: + lines.append("features " + " ".join(sorted(d for d in os.listdir(FEATURES) if d.startswith("network")))) + except OSError as e: + lines.append(f"features unreadable ({e.strerror})") + for s in ("apparmor_restrict_unprivileged_unconfined", "apparmor_restrict_unprivileged_userns"): + read(s, f"/proc/sys/kernel/{s}") + read("core_pattern", "/proc/sys/kernel/core_pattern") + defaults = run("sudo defaults", ["/usr/bin/sudo", "-V"]) + for line in defaults: + low = line.lower() + if "reset the environment" in low or "file descriptors" in low or "closefrom" in low: + lines.append(f"sudo {line.strip()}") + return lines + + def template_bytes(self, path): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + try: + return os.read(fd, 1 << 16) + finally: + os.close(fd) + + def profile_list(self): + return profile_list() + + def kernel_hash(self, name): + return kernel_hash(name) + + def parser_add(self, text): + r = subprocess.run([PARSER, "-a", "-K", "-q"], input=text, capture_output=True, text=True, timeout=15) + if r.returncode != 0: + raise Refused(f"apparmor_parser failed with status {r.returncode}: {(r.stderr or r.stdout).strip()[:500]}") + + def remove(self, name): + fd = os.open(REMOVE, os.O_WRONLY) + try: + os.write(fd, name.encode()) + finally: + os.close(fd) + + def host_pidns(self): + return ns_inode("self", "pid") + + def pidns_tasks(self): + return pidns_tasks() + + def label_sweep(self): + return label_sweep() + + def process_table(self): + return process_table() + + def pidfd_open(self, pid): + return os.pidfd_open(pid) + + def start_time(self, pid): + return parse_stat(read_text(f"/proc/{pid}/stat"))["start"] + + def ns_inodes(self, pid): + """(pid namespace inode, mount namespace inode or None).""" + pid_ino = ns_inode(pid, "pid") + try: + mnt_ino = ns_inode(pid, "mnt") + except FileNotFoundError: + mnt_ino = None # an exiting task loses its mount namespace first + return pid_ino, mnt_ino + + def pidfd_signal(self, pidfd, sig): + signal.pidfd_send_signal(pidfd, sig) + + def close(self, fd): + os.close(fd) + + def sleep(self, seconds): + time.sleep(seconds) + + def clock(self): + return time.monotonic() + + +def preflight(sys_): + good = True + for passed, text in preflight_findings(sys_.preflight_observation()): + say(f"{'ok ' if passed else 'FAIL'} {text}") + good = good and passed + for line in sys_.preflight_record(): + say(f"info {line}") + if not good: + raise Refused("AppArmor cannot confine the launch") + + +def load(op, name, ctl, sys_): + kind, path, sha = template_for(op, name) + text = render(sys_.template_bytes(path), kind, sha, name) + if listed_modes(sys_.profile_list(), name): + raise Refused(f"{name} is loaded already") + sys_.parser_add(text) + modes = listed_modes(sys_.profile_list(), name) + if modes != ["enforce"]: + raise Refused(f"{name} is listed as {modes or 'nothing'} after the load, not once in enforce mode") + h = sys_.kernel_hash(name) + if not ctl.write_new(".hash.load", "hash", f"sha256 {h or 'none'}\n", replace=False): + raise Refused("CTLDIR/hash exists already") + say(f"loaded {name} ({kind} template; kernel hash {h or 'not exported'})") + + +def kill_init_once(launcher, ctl, sys_): + """One attempt: "killed", "none" (not found), "ambiguous" or Refused. The order is the + safety argument: the pid is held by a pidfd and its start time is the + one seen in the lookup, before and after its namespace is read, so the + namespace recorded and the process signalled are those of pid 1.""" + host = sys_.host_pidns() + rec = ctl.record("ns") + recorded = None + if rec is not None: + if p0(rec, host) is not None: + raise Refused(f"CTLDIR/ns is not usable: {p0(rec, host)}") + recorded = parse_ns(rec["text"])[0] + procs = sys_.process_table() + found, what = find_init(procs, launcher, host, recorded) + if found != "found": + return found # "none" or "ambiguous": looked up again until the deadline + pid, start = what, procs[what]["start"] + pidfd = sys_.pidfd_open(pid) + try: + if sys_.start_time(pid) != start: + return "none" + pid_ino, mnt_ino = sys_.ns_inodes(pid) + if sys_.start_time(pid) != start: + return "none" + sys_.pidfd_signal(pidfd, 0) + if pid_ino == host: + raise Refused("pid 1 of the launch is in the host's pid namespace") + if recorded is None: + if not ctl.write_new(".ns.killinit", "ns", ns_record(pid_ino, mnt_ino), replace=False): + rec = ctl.record("ns") + if p0(rec, host) is not None or parse_ns(rec["text"])[0] != pid_ino: + raise Refused("CTLDIR/ns appeared with another namespace; not signalling") + elif recorded != pid_ino: + raise Refused("pid 1 is not in the recorded pid namespace; not signalling") + sys_.pidfd_signal(pidfd, signal.SIGKILL) + say(f"killed pid 1 of the launch (pid {pid}, pid namespace {pid_ino})") + return "killed" + finally: + sys_.close(pidfd) + + +def kill_init(launcher, ctl, sys_): + end = sys_.clock() + KILL_INIT_DEADLINE + while True: + try: + result = kill_init_once(launcher, ctl, sys_) + except (ProcessLookupError, FileNotFoundError): + result = "none" # pid 1 ended while it was being looked at + if result == "killed": + return + if sys_.clock() >= end: + if result == "ambiguous": + raise Refused("pid 1 of the launch could not be told apart from other processes") + say("pid 1 of the launch not found; P1 decides") + return + sys_.sleep(0.1) + + +def release(name, ctl, sys_): + state_rec = ctl.record("state") + state = parse_state(state_rec["text"] if state_rec and state_rec["regular"] else None) + ns_exists = ctl.lstat("ns") is not None + plan = release_plan(state, ns_exists) + if plan == "no-namespace": + if not listed_modes(sys_.profile_list(), name): + say(f"{name} was never loaded; nothing to release (state {state})") + return + else: + host = sys_.host_pidns() + rec = ctl.record("ns") + why = p0(rec, host) + if why: + raise Refused(f"P0: {why}") + why = p1(sys_.pidns_tasks(), parse_ns(rec["text"])[0]) + if why: + raise Refused(f"P1: {why}") + hrec = ctl.record("hash") + hash_load = None + if hrec and hrec["regular"] and hrec["uid"] == 0 and hrec["mode"] == 0o444 and HASH_RE.fullmatch(hrec["text"]): + hash_load = hrec["text"].split()[1] + why = p2(listed_modes(sys_.profile_list(), name), sys_.kernel_hash(name), hash_load) + if why: + raise Refused(f"P2: {why}") + why = sweeps_clear(sys_.label_sweep, name) + if why: + raise Refused(f"sweep: {why}") + sys_.remove(name) + left = listed_modes(sys_.profile_list(), name) + if left: + raise Refused(f"{name} is still listed after its removal ({left})") + say(f"released {name} ({plan}, state {state})") + + +def diag(name, sys_): + try: + say(f"diag list: {listed_modes(sys_.profile_list(), name) or 'not listed'}") + except OSError as e: + say(f"diag list unreadable ({e.strerror})") + try: + carriers, unknown = sweep_result(sys_.label_sweep(), name) + say(f"diag tasks carrying it: {len(carriers)} {' '.join(carriers[:20])}") + if unknown: + say(f"diag labels unreadable: {len(unknown)} {' '.join(unknown[:5])}") + except OSError as e: + say(f"diag sweep failed ({e.strerror})") + + +USAGE = {"preflight": 0, "load": 2, "load-negative-control": 2, "kill-init": 2, "release": 2, "diag": 1} + + +def main(argv, sys_=None): + if not sys.platform.startswith("linux"): + say("Linux only") + return 2 + if len(argv) < 1 or argv[0] not in USAGE or len(argv) - 1 != USAGE[argv[0]]: + say(f"usage: {' | '.join(USAGE)} (got {' '.join(argv) or 'nothing'})") + return 2 + if os.geteuid() != 0: + say("must run as root (through sudo)") + return 2 + sys_ = sys_ or System() + op = argv[0] + try: + if op == "preflight": + preflight(sys_) + elif op in ("load", "load-negative-control"): + if not name_ok(op, argv[1]): + raise Refused(f"{op} does not accept the name {argv[1]!r}") + load(op, argv[1], validated_ctldir(argv[2]), sys_) + elif op == "kill-init": + if not argv[1].isdigit(): + raise Refused(f"LAUNCHERPID {argv[1]!r} is not a pid") + launcher = int(argv[1]) + sudo_uid, _ = sudo_identity() + found = launcher_findings(launcher, sys_.process_table(), os.getpid(), sudo_uid) + if found: + raise Refused(f"LAUNCHERPID {launcher} refused: {'; '.join(found)}") + kill_init(launcher, validated_ctldir(argv[2]), sys_) + elif op == "release": + if not name_ok(op, argv[1]): + raise Refused(f"release does not accept the name {argv[1]!r}") + release(argv[1], validated_ctldir(argv[2]), sys_) + else: + if not name_ok(op, argv[1]): + raise Refused(f"diag does not accept the name {argv[1]!r}") + diag(argv[1], sys_) + except Refused as e: + say(f"refused: {e}") + return 1 + except (OSError, ValueError, subprocess.SubprocessError) as e: + say(f"refused: {op} failed: {type(e).__name__}: {e}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/ci/no-docker-probe.py b/scripts/ci/no-docker-probe.py new file mode 100644 index 0000000..06dde21 --- /dev/null +++ b/scripts/ci/no-docker-probe.py @@ -0,0 +1,1691 @@ +"""Prove that this process is isolated by scripts/ci/no-docker.sh. + +Run by no-docker.sh inside its isolation, before the command it protects +(macOS: from --inner; Linux: as --gate-exec, the process that becomes the +command), and by `no-docker.sh --check`. Exit 0 only if + + - every mechanism of the isolation is in effect in this process: + macOS: a connect to a Unix socket of our own, a TCP connect to + loopback port 2375 and to a non-loopback address are refused by the + sandbox (EPERM), not merely unanswered; no loopback port that was + listening before the sandbox started (NO_DOCKER_HELD_PORTS) can be + connected to; no signal reaches no-docker.sh outside the sandbox + (NO_DOCKER_OUTER_PID, EPERM); no file can be created under $HOME, in + the user's temporary directory (NO_DOCKER_USER_TMP), in /tmp or in + /private/var/tmp, while $TMPDIR (inside the run's directory) is + writable; + Linux: not uid 0, no capability in any set, no_new_privs, no + supplementary group, a primary group that is not a system group; + network, mount, pid and IPC namespaces that differ from the caller's + (NO_DOCKER_HOST_NS); a pid namespace below the caller's (the caller's + is the initial one, ours is not), with a /proc of its own (NSpid is + our pid alone, /proc/self is our pid) that hides no process, whose + pid 1 is the root shell of no-docker.sh and whose other processes + are all ours; a session of our own without a controlling terminal; + standard descriptors that are pipes or /dev/null; only the loopback + interface; /tmp, /var/tmp, + /dev/shm, /run and /dev/mqueue are new file systems + (NO_DOCKER_HOST_DEVS) and /sys/fs/cgroup is masked; every other + mount reachable by path is read-only; this process, and every other + one but the root pid 1, confined by exactly the AppArmor profile of + the launch in enforce mode (NO_DOCKER_PROFILE); writes to $HOME, + $GITHUB_WORKSPACE and $RUNNER_TEMP (NO_DOCKER_PROTECTED) fail; no + cgroup file is writable; and no socket in the private directories. + As --gate-exec, in addition: the namespace record of the launch + (NO_DOCKER_CTL/ns) names this pid namespace and the control and + fence directories are read-only; the two sockets no-docker.sh bound + outside (NO_DOCKER_FENCE), which file permissions let us reach, + refuse a connect and a datagram with EACCES, while a socket bound + inside connects; and, last, the descriptors are exactly /dev/null + on fd 0 and pipes on fds 1 and 2. Only then are the signal + dispositions reset, the start recorded and the command run in this + process. Host-bound Unix sockets are not listed: the kernel denies + them (scripts/ci/no-docker/apparmor.profile); + - and no container engine answers on any endpoint we can find: the + usual socket paths of Docker Engine, Docker Desktop, rootless Docker, + Podman, Colima, OrbStack and Rancher Desktop; every endpoint of the + Docker contexts under ~/.docker; DOCKER_HOST and the DOCKER_HOST the + caller of no-docker.sh had (NO_DOCKER_ORIGINAL_HOST); every socket under + /run, /var/run, /tmp, $TMPDIR and ~/.docker; loopback ports 2375 and + 2376; and every listening TCP port. + +An endpoint is an engine if it accepts a connection and answers +`GET /_ping` (sent with the User-Agent no-docker-probe) with an API version +header. Nothing is written, connected to or sent to an engine before the +mechanism checks passed. Prints one line per check. + +Modes used by no-docker.sh: + --listening print the listening TCP port numbers (macOS), one per line; + --root-exec UID GID NAME -- NAME=VALUE... -- COMMAND... + as root inside the namespaces (Linux): close every + descriptor above 2 and show that none is left, open + /dev/null as fd 0, then exec setpriv (uid and gid, no groups, + capabilities or new privileges), aa-exec -p NAME, setsid and + env -i with the given environment, which run --gate-exec; + --gate-exec -- COMMAND... + the checks above, then COMMAND in this process (Linux; reports + on fd 2, 125 if refused); + --seal WORK as root in the new mount namespace: make every mount + reachable by path read-only, except the private file systems + and WORK, and verify it (Linux); + --markers TMP OWNER + make a marker directory for a new run in the per-user + temporary directory TMP, recording the process OWNER that + supervises it, and print its path (macOS); + --orphans TMP fail if a run that ended without its supervisor may still + have processes (macOS); + --supervise MARKDIR STARTED REMOVE -- COMMAND... + run COMMAND (sandbox-exec), then end every process left in + the sandbox of MARKDIR; signals end it at once; 125 unless + the file STARTED exists; removes MARKDIR and the directory + REMOVE ("-": none) (macOS, no-docker.sh's last step); + --end MARKDIR end every process of the sandbox of MARKDIR, and remove it + once that is shown (macOS). +""" + +import concurrent.futures +import ctypes +import errno +import glob +import grp +import json +import os +import re +import resource +import shutil +import signal +import socket +import stat +import struct +import subprocess +import sys +import tempfile +import time + +ok = True + + +def report(passed, text): + global ok + if not passed: + ok = False + print(f"no-docker: {'ok ' if passed else 'FAIL'} {text}") + + +def denied(family, address): + """True if connecting is refused by the sandbox (EPERM/EACCES).""" + s = socket.socket(family, socket.SOCK_STREAM) + s.settimeout(2) + try: + s.connect(address) + except OSError as e: + return e.errno in (errno.EPERM, errno.EACCES) + finally: + s.close() + return False + + +def engine_answers(family, address, timeout=2): + """Return a description if an engine answers at address, else None.""" + s = socket.socket(family, socket.SOCK_STREAM) + s.settimeout(timeout) + try: + s.connect(address) + s.sendall(b"GET /_ping HTTP/1.0\r\nHost: docker\r\nUser-Agent: no-docker-probe\r\n\r\n") + data = b"" + # Stop at the end of the headers: an engine may keep the connection + # open (HTTP/1.1 keep-alive) instead of closing it after the answer. + while len(data) < 4096 and b"\r\n\r\n" not in data: + try: + chunk = s.recv(4096) + except OSError: + break + if not chunk: + break + data += chunk + except OSError: + return None + finally: + s.close() + head = data.split(b"\r\n\r\n", 1)[0].lower() + if b"api-version:" in head or b"libpod-api-version:" in head: + return head.split(b"\r\n", 1)[0].decode("ascii", "replace") + return None + + +def connects(address): + """True if a TCP connect succeeds; nothing is sent.""" + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(1) + try: + s.connect(address) + except OSError: + return False + finally: + s.close() + return True + + +def create_error(directory): + """None if a new file can be created in directory (it is removed again), + else the errno of the failure.""" + return create_error_named(directory, f".no-docker-probe-{os.getpid()}") + + +def create_error_named(directory, name): + """As create_error, for a file of the given name.""" + path = os.path.join(directory, name) + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + except FileExistsError: + return None + except OSError as e: + return e.errno + os.close(fd) + os.unlink(path) + return None + + +def not_writable(directory): + return create_error(directory) is not None + + +class WalkLimit(Exception): + pass + + +def walk_sockets(roots, limit=20000): + """Socket paths under roots; raises WalkLimit instead of stopping + silently when more than limit entries would have to be looked at.""" + found = [] + seen = 0 + for root in roots: + if not root or not os.path.isdir(root): + continue + for dirpath, dirnames, filenames in os.walk(root, onerror=lambda e: None): + for name in dirnames + filenames: + seen += 1 + if seen > limit: + raise WalkLimit(f"more than {limit} entries under {' '.join(r for r in roots if r)}") + p = os.path.join(dirpath, name) + try: + if stat.S_ISSOCK(os.lstat(p).st_mode): + found.append(p) + except OSError: + pass + return found + + +# --- macOS ------------------------------------------------------------------- + +def check_mechanism_darwin(): + d = tempfile.mkdtemp(prefix="no-docker-probe.") + # Bound and connected by a relative path: the absolute one can exceed + # the 104 bytes of sun_path. + cwd = os.open(".", os.O_RDONLY) + srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + os.chdir(d) + srv.bind("s") + srv.listen(1) + report(denied(socket.AF_UNIX, "s"), "macOS sandbox denies Unix socket connects") + finally: + srv.close() + try: + os.unlink("s") + except OSError: + pass + os.fchdir(cwd) + os.close(cwd) + os.rmdir(d) + report(denied(socket.AF_INET, ("127.0.0.1", 2375)), "macOS sandbox denies loopback port 2375") + report(denied(socket.AF_INET, ("192.0.2.1", 2375)), "macOS sandbox denies non-loopback TCP") + if not ok: + # Not sandboxed: write nothing, signal nothing and connect to nothing else. + return + held = os.environ.get("NO_DOCKER_HELD_PORTS", "").split() + reached = [p for p in held if p.isdigit() and connects(("127.0.0.1", int(p)))] + report(not reached, f"macOS sandbox denies the {len(held)} loopback ports listening before it started" + + (": reached " + " ".join(reached) if reached else "")) + outer = os.environ.get("NO_DOCKER_OUTER_PID", "") + result = "unknown" + if outer.isdigit(): + try: + os.kill(int(outer), 0) + result = "reached" + except PermissionError: + result = "EPERM" + except OSError as e: + result = errno.errorcode.get(e.errno, str(e.errno)) + report(result == "EPERM", f"macOS sandbox denies signals to processes outside it (no-docker.sh {outer or '?'}: {result})") + home = os.path.realpath(os.path.expanduser("~")) + user_tmp = os.environ.get("NO_DOCKER_USER_TMP", "") + report(bool(user_tmp), "the user's temporary directory is known (NO_DOCKER_USER_TMP)") + for where, d in (("under $HOME", home), ("in the user's temporary directory", user_tmp), + ("in /tmp", "/private/tmp"), ("in /private/var/tmp", "/private/var/tmp")): + if d: + report(not_writable(d), f"macOS sandbox denies writes {where} ({d})") + tmpdir = os.environ.get("TMPDIR", "") + report(bool(tmpdir) and create_error(tmpdir) is None, f"$TMPDIR of the run is writable ({tmpdir})") + # bash 3.2 keeps here-documents in /private/tmp/sh-thd-*: none of another + # bash outside may be created, replaced or rewritten from here. + e = create_error_named("/private/tmp", f"sh-thd-no-docker-probe-{os.getpid()}") + report(e is not None, "macOS sandbox denies writes to here-document files of bash 3.2 (/private/tmp/sh-thd-*)" + + ("" if e is not None else ": written")) + # The markers by which no-docker.sh finds this sandbox's processes from + # outside: reading A is denied, reading B is not. + for d in (os.environ.get("NO_DOCKER_MARK", ""),): + result = "unknown" + if d: + try: + open(os.path.join(d, "b"), "rb").close() + try: + open(os.path.join(d, "a"), "rb").close() + result = "not denied" + except PermissionError: + result = "ok" + except OSError as err: + result = errno.errorcode.get(err.errno, str(err)) + report(result == "ok", f"the sandbox carries the marker of {d or '?'} ({result})") + + +# The processes of one sandbox, seen from outside it. no-docker.sh (and the +# suite's own guard) add to the profile `(deny file-read-data (literal A))` +# for file A of the run's marker directory and leave file B next to it +# alone; no other profile names these files. sandbox_check() asks the +# kernel how a process's sandbox would decide, so a process is in the +# run's sandbox exactly when it is sandboxed at all, denied A and allowed +# B (measured: an unsandboxed process is allowed both; the files must +# exist, or every process is "denied"). A process cannot leave its +# sandbox, and every process it starts inherits it. The marker directory +# is in the per-user temporary directory, outside every directory the +# sandbox may write, so the command cannot remove or replace it; it also +# records the run's owner (the supervisor), so a run that ended without +# its supervisor can be recognised later. +MARK_PREFIX = "no-docker-mark." +SZOMB = 5 + + +class ProcShortInfo(ctypes.Structure): + # struct proc_bsdshortinfo (PROC_PIDT_SHORTBSDINFO): answers for + # processes of other users too. + _fields_ = [("pid", ctypes.c_uint32), ("ppid", ctypes.c_uint32), ("pgid", ctypes.c_uint32), + ("status", ctypes.c_uint32), ("comm", ctypes.c_char * 16), ("flags", ctypes.c_uint32), + ("uid", ctypes.c_uint32), ("gid", ctypes.c_uint32), ("ruid", ctypes.c_uint32), + ("rgid", ctypes.c_uint32), ("svuid", ctypes.c_uint32), ("svgid", ctypes.c_uint32), + ("rfu", ctypes.c_uint32)] + + +class ProcUniqIdentifierInfo(ctypes.Structure): + # struct proc_uniqidentifierinfo (PROC_PIDUNIQIDENTIFIERINFO): an id + # the kernel never reuses while it runs. + _fields_ = [("uuid", ctypes.c_uint8 * 16), ("uniqueid", ctypes.c_uint64), ("puniqueid", ctypes.c_uint64), + ("idversion", ctypes.c_int32), ("orig_ppidversion", ctypes.c_uint32), + ("reserve3", ctypes.c_uint64), ("reserve4", ctypes.c_uint64)] + + +def private_dir(path): + """True if path is a directory of ours, mode 0700, not a symbolic link.""" + st = os.lstat(path) + return stat.S_ISDIR(st.st_mode) and st.st_uid == os.getuid() and stat.S_IMODE(st.st_mode) == 0o700 + + +def regular_file(path): + st = os.lstat(path) + return stat.S_ISREG(st.st_mode) and st.st_uid == os.getuid() + + +def marker_files(directory): + """The marker files A and B of directory, which must be a private + directory with two empty regular files of ours.""" + if not private_dir(directory): + raise OSError(f"{directory} is not a directory of ours with mode 0700") + files = [] + for name in ("a", "b"): + path = os.path.join(directory, name) + if not regular_file(path): + raise OSError(f"{path} is not a regular file of ours") + files.append(os.fsencode(path)) + return files + + +class Gone(Exception): + """The process ended (or its pid now names another process).""" + + +class ProcTable: + """The process table (libproc). Raises OSError if it is unavailable or + gives an answer that cannot be interpreted, so every caller fails + closed.""" + + PROC_PIDT_SHORTBSDINFO = 13 + PROC_PIDUNIQIDENTIFIERINFO = 17 + + def __init__(self): + try: + self.proc = ctypes.CDLL("/usr/lib/libproc.dylib", use_errno=True) + except OSError as e: + raise OSError(f"libproc is not available: {e}") from None + self.proc.proc_listallpids.argtypes = [ctypes.c_void_p, ctypes.c_int] + self.proc.proc_pidinfo.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64, ctypes.c_void_p, ctypes.c_int] + + def snapshot(self): + """Every pid, from one copy of the kernel's process lists (taken + under one lock, so it holds every process that existed at that + instant, zombies included). A full buffer may mean a cut list, so + it is taken again with a larger one.""" + for _ in range(8): + n = self.proc.proc_listallpids(None, 0) + if n <= 0: + raise OSError("cannot list the processes") + room = n + 4096 + buf = (ctypes.c_int * room)() + got = self.proc.proc_listallpids(buf, ctypes.sizeof(buf)) + if got <= 0: + raise OSError("cannot list the processes") + if got < room: + return [buf[i] for i in range(got) if buf[i] > 0 and buf[i] != os.getpid()] + raise OSError("the process list did not fit") + + def _info(self, pid, flavor, struct, arg=0): + ctypes.set_errno(0) + n = self.proc.proc_pidinfo(pid, flavor, arg, ctypes.byref(struct), ctypes.sizeof(struct)) + if n == ctypes.sizeof(struct): + return True + if ctypes.get_errno() in (errno.ESRCH, 0): + return False + raise OSError(f"cannot read process {pid} ({os.strerror(ctypes.get_errno())})") + + def identity(self, pid): + """The kernel's unique id of the process, or None if it is gone (or + a zombie).""" + u = ProcUniqIdentifierInfo() + return u.uniqueid if self._info(pid, self.PROC_PIDUNIQIDENTIFIERINFO, u) else None + + def parent_identity(self, pid): + """The unique id of the process's parent, or None if it is gone.""" + u = ProcUniqIdentifierInfo() + return u.puniqueid if self._info(pid, self.PROC_PIDUNIQIDENTIFIERINFO, u) else None + + def zombie(self, pid): + """True if pid is a zombie (only an argument of 1 finds those).""" + s = ProcShortInfo() + return self._info(pid, self.PROC_PIDT_SHORTBSDINFO, s, arg=1) and s.status == SZOMB + + +class Sandboxes(ProcTable): + """The process table and sandbox_check() for the marker of one run. + Raises OSError if either is unavailable or gives an answer it cannot + interpret, so every caller fails closed.""" + + FILTER_PATH = 1 + + def __init__(self, directory): + super().__init__() + self.directory = directory + self.deny, self.allow = marker_files(directory) + try: + sb = ctypes.CDLL("/usr/lib/system/libsystem_sandbox.dylib", use_errno=True) + self.check_fn = sb.sandbox_check + no_report = ctypes.c_int.in_dll(sb, "SANDBOX_CHECK_NO_REPORT").value + except (OSError, AttributeError, ValueError) as e: + raise OSError(f"sandbox_check() is not available: {e}") from None + # Variadic after the third parameter: with exactly three argtypes, + # ctypes passes the path as a variadic argument (on the stack on + # arm64), as the function expects. + self.check_fn.restype = ctypes.c_int + self.check_fn.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int] + self.kind = self.FILTER_PATH | no_report + self.verify() + if self.identity(os.getpid()) is None: + raise OSError("process identities cannot be read") + + def verify(self): + """The marker files still exist and this (unsandboxed) process is + allowed both and is not sandboxed. A missing file reads as denied + for every process, so this runs before every scan, after every + membership answer and before every signal.""" + marker_files(self.directory) + me = os.getpid() + if (self._check(me, self.deny), self._check(me, self.allow), self.check_fn(me, None, 0)) != (0, 0, 0): + raise OSError("sandbox_check() does not answer as expected for this process") + + def _check(self, pid, path): + return self.check_fn(pid, b"file-read-data", self.kind, ctypes.c_char_p(path)) + + def classify(self, pid): + """("member" | "other" | "zombie", unique id) of pid; raises Gone if + it ended meanwhile, OSError if it cannot be told. Membership is read + between two identity reads, so the answer belongs to one process.""" + first = self.identity(pid) + if first is None: + if self.zombie(pid): + return "zombie", None + raise Gone() + s = ProcShortInfo() + if not self._info(pid, self.PROC_PIDT_SHORTBSDINFO, s): + raise Gone() + if s.status == SZOMB: + return "zombie", first + sandboxed = self.check_fn(pid, None, 0) + a, b = self._check(pid, self.deny), self._check(pid, self.allow) + self.verify() + if self.identity(pid) != first: + raise Gone() + if sandboxed not in (0, 1) or a not in (0, 1) or b not in (0, 1): + raise OSError(f"sandbox_check() failed for pid {pid} ({sandboxed}, {a}, {b})") + return ("member" if sandboxed == 1 and a == 1 and b == 0 else "other"), first + + +def sweep(sbx, seconds, kill, owner=None): + """Show that no process of the sandbox runs any more, killing each one + found if kill (SIGKILL, after its identity and the markers were read + again). Returns (ended, members) once a round proves it: in one + snapshot every pid is a process that is not a member (membership never + changes), or one already dead (a zombie or gone) before that snapshot + was taken. Every process started later descends from a process in the + snapshot (a child is listed before it can run, its parent until it is + reaped), so none of them can be a member either. A pid that ended + before it could be told apart, or became a zombie only now, leaves the + round open: it may have started a member before it died. Raises + OSError when no round proves it within seconds (for example a process + that keeps starting new ones and ending); the caller fails closed. + owner (the unique id of a run's owner that is gone): a process the + owner started that is not a member (yet) keeps every round open, as it + may be the run's sandbox-exec before it applied the profile, the one + process that becomes a member after it was checked. Relies on pids not + being reused within one round (the kernel's counter would have to wrap + at 99999).""" + deadline = time.monotonic() + seconds + others = {} # pid: unique id of a process that is not a member + dead = set() # pids seen dead before the current snapshot + ended = 0 + members = set() + rounds = 0 + while True: + rounds += 1 + sbx.verify() + snap = sbx.snapshot() + present = set(snap) + proven = True + dead_now = set() + for pid in snap: + if pid in dead: + continue + try: + kind, uid = sbx.classify(pid) + except Gone: + if pid not in others: + proven = False + continue + if kind == "other": + if owner is not None and sbx.parent_identity(pid) == owner: + proven = False + continue + others[pid] = uid + elif kind == "zombie": + dead_now.add(pid) + if pid not in others: + proven = False + else: + proven = False + members.add(pid) + if kill and sbx.identity(pid) == uid: + sbx.verify() + try: + os.kill(pid, signal.SIGKILL) + ended += 1 + except ProcessLookupError: + pass + except PermissionError: + pass # another user's (a setuid program): still found next round + if proven: + return ended, members + # A pid absent from a snapshot may name a new process later. + dead = (dead | dead_now) & present + others = {p: u for p, u in others.items() if p in present} + if time.monotonic() > deadline: + live = sorted(p for p in members if sbx.identity(p) is not None) + raise OSError(f"cannot show within {seconds:.0f}s that every process of the isolation ended " + f"({rounds} rounds; still running: {' '.join(map(str, live)) or 'none seen'})") + time.sleep(0.005) + + +def owner_record(directory): + """(pid, unique id) the marker directory records as its owner; raises + OSError if the record is unreadable.""" + rec = os.path.join(directory, "owner") + if not regular_file(rec) or os.lstat(rec).st_size > 64: + raise OSError(f"{rec} is not a regular file of ours") + with open(rec) as f: + fields = f.read().split() + if len(fields) != 2 or not fields[0].isdigit() or not fields[1].isdigit(): + raise OSError(f"{rec} is not an owner record") + return int(fields[0]), int(fields[1]) + + +def owner_alive(table, directory): + """True if the owner the marker directory records still runs (same pid + and unique id), False if it is gone; raises OSError if the record is + unreadable.""" + pid, uid = owner_record(directory) + return table.identity(pid) == uid + + +def darwin_markers(tmp, owner): + """For no-docker.sh and the suite: make a marker directory for a new run + in the per-user temporary directory tmp, recording its owner (pid and + unique id; the supervisor that owner becomes, or the suite), and print + its path.""" + run = tempfile.mkdtemp(prefix=MARK_PREFIX, dir=tmp) + for name in ("a", "b"): + os.close(os.open(os.path.join(run, name), os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)) + table = ProcTable() + uid = table.identity(int(owner)) if owner.isdigit() else None + if uid is None: + shutil.rmtree(run, ignore_errors=True) + raise OSError(f"the owner {owner} of the run does not exist") + # Complete or absent for a reader: written aside, then renamed. + part = os.path.join(run, "owner.part") + fd = os.open(part, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as f: + f.write(f"{owner} {uid}\n") + os.rename(part, os.path.join(run, "owner")) + print(run) + return 0 + + +def darwin_orphans(tmp, seconds=5.0): + """Before a sandbox starts: refuse if an earlier run ended without its + supervisor (no-docker.sh killed, or its ending failed) and processes of + it may still run. Each run's marker directory records its owner; one + whose owner is gone is checked with sweep(), without killing: proven + empty, it is removed; otherwise (members found, or not provable in + time) the run is refused and they are named. Directories of runs whose + owner lives (other runs in progress) are left alone.""" + table = ProcTable() + stale = [] + for name in sorted(os.listdir(tmp)): + if not name.startswith(MARK_PREFIX): + continue + d = os.path.join(tmp, name) + try: + if not private_dir(d): + continue + if not os.path.lexists(os.path.join(d, "owner")): + # Renamed into place right after the directory is made; an + # old one without it is a run of an earlier version. + if time.time() - os.lstat(d).st_mtime < 60: + continue + stale.append(d) + continue + if not owner_alive(table, d): + stale.append(d) + except FileNotFoundError: + continue # removed meanwhile by its own run + except OSError: + stale.append(d) + left = [] + for d in stale: + if not os.path.lexists(d): + continue # removed meanwhile by its own run + try: + try: + gone_owner = owner_record(d)[1] + except FileNotFoundError: + gone_owner = None + _, members = sweep(Sandboxes(d), seconds, kill=False, owner=gone_owner) + except OSError as e: + # Among others: a marker file is missing, so its processes + # cannot be found any more; the directory stays until someone + # who knows that none runs removes it. + left.append(f"{d}: {e}") + continue + if members: + left.append(f"{d}: processes {' '.join(map(str, sorted(members)))}") + else: + shutil.rmtree(d, ignore_errors=True) + if left: + print("no-docker: an earlier isolated run ended without its supervisor and may have processes left; " + "end them with: python3 -I " + os.path.abspath(__file__) + " --end : " + + "; ".join(left), file=sys.stderr) + return 1 + return 0 + + +def darwin_end(sbx, seconds=10.0): + """End every process of the run's sandbox (sweep with kill).""" + return sweep(sbx, seconds, kill=True)[0] + + +class Interrupted(Exception): + pass + + +def darwin_supervise(markdir, started_file, remove, argv): + """no-docker.sh's last step on macOS, in its process: run argv (the + sandbox-exec command line), wait for it, then end whatever is left in + its sandbox; SIGINT, SIGTERM, SIGHUP or SIGQUIT end it at once the same + way. Everything that needs a library is loaded before the command + starts, and ending it needs no new process, so a command that uses up + the processes it may start cannot prevent it. The run succeeds only + when a round of sweep() proved that none of its processes runs any + more. Removes the run's marker directory (kept when that could not be + shown, so that the next run refuses and the processes can still be + ended through it) and the directory remove (if not "-"). The status is + the command's, or 125 if the isolation could not be applied + (started_file, which the command's start creates, is missing) or its + processes could not be shown to have ended; after a signal, 128 plus + its number.""" + def refused(why): + print(f"no-docker: refused: {why}", file=sys.stderr) + return 125 + + try: + sbx = Sandboxes(markdir) + except OSError as e: + shutil.rmtree(markdir, ignore_errors=True) + if remove != "-": + shutil.rmtree(remove, ignore_errors=True) + return refused(f"cannot identify the processes of the isolation: {e}; the command was not run") + caught = [] + waiting = [False] + + def on_signal(signum, frame): + # Only the wait for the command is cut short; ending the processes + # always runs to its end. + caught.append(signum) + if waiting[0]: + waiting[0] = False + raise Interrupted() + + # A signal the caller ignores stays ignored, for the command too. + for s in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT): + if signal.getsignal(s) != signal.SIG_IGN: + signal.signal(s, on_signal) + child = None + status = 125 + ended, failure = 0, None + try: + try: + child = subprocess.Popen(argv, close_fds=False) + except OSError as e: + print(f"no-docker: cannot start the isolation: {e}", file=sys.stderr) + try: + try: + # A signal that came while it was being started is not + # lost: the wait is skipped. + waiting[0] = True + if child is not None and not caught: + status = child.wait() + finally: + waiting[0] = False + except Interrupted: + # Also when it arrives in the finally clause above (the handler + # clears waiting before it raises, so it raises only once). + pass + if child is not None: + if child.poll() is None: + # Interrupted: our own child, not yet reaped, so its pid is + # still its own (also when sandbox-exec has not applied the + # profile yet). + child.kill() + status = child.wait(timeout=10) + ended = darwin_end(sbx) + except BaseException as e: # noqa: BLE001 - every failure fails closed + failure = str(e) or type(e).__name__ + finally: + if child is not None and child.poll() is None: + child.kill() + try: + child.wait(timeout=10) + except subprocess.TimeoutExpired: + failure = failure or f"the isolation tool (pid {child.pid}) does not end" + started = os.path.exists(started_file) + if not failure: + shutil.rmtree(markdir, ignore_errors=True) + if remove != "-": + shutil.rmtree(remove, ignore_errors=True) + if ended: + print(f"no-docker: ended {ended} processes the command left running", file=sys.stderr) + if failure: + return refused(f"the processes of the isolated command could not be shown to have ended: {failure}; " + f"end them with: python3 -I {os.path.abspath(__file__)} --end {markdir}") + if not started: + return refused(f"the isolation tool returned {status} without starting the command; the command was not run") + if caught: + print(f"no-docker: interrupted by signal {caught[0]}; every process of the command was ended", file=sys.stderr) + return 128 + caught[0] + return 128 - status if status < 0 else status + + +# --- Linux ------------------------------------------------------------------- + +# Mounts that stay writable inside: the private file systems, /proc and /dev +# (device nodes; no file can be created there without root). +WRITABLE = ("/tmp", "/var/tmp", "/dev/shm", "/run", "/var/run", "/dev/mqueue", "/proc", "/dev", "/dev/pts") +PRIVATE = (("/tmp", "tmpfs"), ("/var/tmp", "tmpfs"), ("/dev/shm", "tmpfs"), ("/run", "tmpfs"), + ("/dev/mqueue", "mqueue")) +HOST_COMMS = {"systemd", "init", "Runner.Worker", "Runner.Listener", "dockerd", "containerd", "sshd", + "containerd-shim", "rootlesskit", "podman", "dbus-daemon"} +# The initial pid namespace, from which every other one descends: its inode +# number is fixed (PROC_PID_INIT_INO); every other namespace gets a number +# of 0xF0000000 or above. +INIT_PID_NS = "pid:[4026531836]" +# Debian and Ubuntu give system groups (docker, disk, adm, shadow, +# systemd-journal, ...) gids up to 999. +SYS_GID_MAX = 999 +# The AppArmor profile of a launch (no-docker-aa.py) and the namespace +# record pid 1 writes (CTLDIR/ns: "pid: mnt:"). +NAME_RE = re.compile(r"flowcrm-nd-(neg-)?[0-9a-f]{32}") +NS_RE = re.compile(r"pid:([1-9][0-9]{0,19}) mnt:([1-9][0-9]{0,19}|-)\n") +ENV_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*=.*", re.S) +# The chain --root-exec starts: credentials dropped, then the profile +# attached at the next exec, a session of its own, the allowlisted +# environment and the gate. +SETPRIV, AA_EXEC, SETSID, ENV, PYTHON = "/usr/bin/setpriv", "/usr/bin/aa-exec", "/usr/bin/setsid", "/usr/bin/env", \ + "/usr/bin/python3" +EX_REFUSED = 125 + + +def unescape(s): + return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), s) + + +def mounts(): + entries = [] + with open("/proc/self/mountinfo") as f: + for line in f: + fields = line.rstrip("\n").split(" ") + sep = fields.index("-") + entries.append({"id": int(fields[0]), "point": unescape(fields[4]), + "opts": fields[5].split(","), "fstype": fields[sep + 1], + "super": fields[sep + 3].split(",") if len(fields) > sep + 3 else []}) + return entries + + +def mount_id_of(path): + """The mount id of the file at path (not following a final symbolic + link), or None if path cannot be reached.""" + try: + fd = os.open(path, os.O_PATH | os.O_NOFOLLOW) + except OSError as e: + if e.errno in (errno.ENOENT, errno.EACCES, errno.ENOTDIR, errno.ELOOP): + return None + raise + try: + with open(f"/proc/self/fdinfo/{fd}") as f: + for line in f: + if line.startswith("mnt_id:"): + return int(line.split()[1]) + finally: + os.close(fd) + raise OSError(f"no mnt_id for {path}") + + +def visible_mounts(): + """The mounts reachable by path: a mount hidden under another one, or + under a path that no longer leads to it, cannot be written through.""" + return [m for m in mounts() if mount_id_of(m["point"]) == m["id"]] + + +def writable_mounts(work): + keep = set(WRITABLE) | {work} + return [m["point"] for m in visible_mounts() if m["point"] not in keep and "ro" not in m["opts"]] + + +MS_RDONLY, MS_NOSUID, MS_NODEV, MS_NOEXEC, MS_REMOUNT = 1, 2, 4, 8, 32 +MS_NOSYMFOLLOW, MS_NOATIME, MS_NODIRATIME, MS_BIND = 256, 1024, 2048, 4096 +MS_RELATIME, MS_STRICTATIME = 1 << 21, 1 << 24 +KEEP_FLAGS = {"nosuid": MS_NOSUID, "nodev": MS_NODEV, "noexec": MS_NOEXEC, "nosymfollow": MS_NOSYMFOLLOW, + "noatime": MS_NOATIME, "nodiratime": MS_NODIRATIME, "relatime": MS_RELATIME, + "strictatime": MS_STRICTATIME} + + +def seal(work): + """As root in the new mount namespace: remount every reachable mount + except WRITABLE and work read-only (a bind remount changes only this + namespace's copy and keeps its other per-mount flags), then verify.""" + libc = ctypes.CDLL(None, use_errno=True) + libc.mount.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulong, ctypes.c_void_p] + keep = set(WRITABLE) | {work} + count = 0 + for m in visible_mounts(): + if m["point"] in keep or "ro" in m["opts"]: + continue + flags = MS_REMOUNT | MS_BIND | MS_RDONLY + for o in m["opts"]: + flags |= KEEP_FLAGS.get(o, 0) + if libc.mount(None, os.fsencode(m["point"]), None, flags, None) != 0: + e = ctypes.get_errno() + print(f"no-docker: cannot make {m['point']} read-only: {os.strerror(e)}", file=sys.stderr) + return 1 + count += 1 + left = writable_mounts(work) + if left: + print(f"no-docker: still writable after the remount: {' '.join(left)}", file=sys.stderr) + return 1 + print(f"no-docker: {count} mounts made read-only", file=sys.stderr) + return 0 + + +def gone(e): + """True for the errors of a process or fd that ended meanwhile.""" + return e.errno in (errno.ENOENT, errno.ESRCH) + + +def status(pid="self"): + st = {} + with open(f"/proc/{pid}/status") as f: + for line in f: + k, _, v = line.partition(":") + st[k] = v.strip() + return st + + +def env_pairs(name): + return dict(x.split("=", 1) for x in os.environ.get(name, "").split() if "=" in x) + + +def eval_creds(st, group): + """Findings on this process's credentials (st: /proc/self/status).""" + uids = st.get("Uid", "0").split() + caps = {k: st.get(k, "unknown") for k in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb")} + + def zero(v): + return re.fullmatch(r"[0-9a-f]+", v) is not None and int(v, 16) == 0 + + gids = st.get("Gid", "").split() + plain = len(gids) == 4 and len(set(gids)) == 1 and gids[0].isdigit() and int(gids[0]) > SYS_GID_MAX + return [ + ("0" not in uids, f"not root (uids {' '.join(uids)})"), + (all(zero(v) for v in caps.values()), "no capabilities (" + " ".join(f"{k}={v}" for k, v in caps.items()) + ")"), + (st.get("NoNewPrivs") == "1", f"no_new_privs is set ({st.get('NoNewPrivs', 'unknown')})"), + (st.get("Groups", "x") == "", f"no supplementary groups ({st.get('Groups') or 'none'})"), + # The primary group is the caller's: not one of the system groups, + # which include the one of the engine socket. + (plain, f"the primary group is not a system group (gids {' '.join(gids) or 'unknown'}; {group})"), + ] + + +def eval_pidns(nspid, pid, own, host_ns, proc_self): + """Findings on the pid namespace. host_ns: NO_DOCKER_HOST_NS as given.""" + callers = [x[len("pid="):] for x in host_ns.split() if x.startswith("pid=")] + caller = callers[0] if len(callers) == 1 else "" + well_formed = re.fullmatch(r"pid:\[\d+\]", own) is not None + return [ + (caller == INIT_PID_NS, f"the caller is in the initial pid namespace (caller {caller or 'unknown'})"), + # Every pid namespace descends from the initial one: one that is + # not it is below the caller's. + (well_formed and own != INIT_PID_NS and own != caller, + f"in a pid namespace below the caller's ({own}; caller {caller or 'unknown'})"), + # A procfs shows NSpid from its own namespace down: one entry, this + # pid, in the namespace's own /proc; more from an ancestor's. + (nspid.split() == [str(pid)], f"/proc shows this process in its own pid namespace only (NSpid {nspid or 'missing'})"), + (proc_self == str(pid), "/proc belongs to this pid namespace"), + ] + + +def eval_processes(entries, me): + """entries: (pid, its status or the OSError reading it) for every pid in + /proc. Only an ended process may be skipped: one that cannot be read + (hidepid) counts as foreign, and pid 1 must have been read.""" + foreign = [] + count = 0 + root = False + for d, st in entries: + if isinstance(st, OSError): + if not gone(st): + foreign.append(f"{d}:unreadable({errno.errorcode.get(st.errno, st.errno)})") + continue + count += 1 + name, puids = st.get("Name", "?"), st.get("Uid", "").split() + if d == "1": + root = puids[:1] == ["0"] and name in ("sh", "dash") + if not root: + foreign.append(f"1:{name}:{'/'.join(puids)}") + elif not puids or any(u != str(me) for u in puids) or name in HOST_COMMS: + foreign.append(f"{d}:{name}:{'/'.join(puids)}") + if not root and not any(f.startswith("1:") for f in foreign): + foreign.append("1:missing") + return [(not foreign, f"only our processes and the root pid 1 of the isolation are visible ({count} processes" + + (f"; foreign: {' '.join(foreign)}" if foreign else "") + ")")] + + +def eval_mounts(vis, host_devs, devs, work, left): + """vis: the reachable mounts by mount point; devs: st_dev of the private + paths; left: the mounts still writable (None without a run directory).""" + f = [] + proc = vis.get("/proc") + hidden = [o for o in (proc or {}).get("opts", []) + (proc or {}).get("super", []) + if (o.startswith("hidepid=") and o not in ("hidepid=0", "hidepid=off")) or o.startswith("subset=")] + f.append((bool(proc) and proc["fstype"] == "proc" and not hidden, + f"/proc shows every process of the namespace ({','.join(hidden) or 'no hidepid or subset'})")) + for path, fstype in PRIVATE: + m = vis.get(path) + dev = devs.get(path, "?") + host = host_devs.get(path, "") + f.append((bool(m) and m["fstype"] == fstype and bool(host) and dev != host, + f"{path} is a private {fstype} (device {dev}; caller {host or 'unknown'})")) + # A second procfs, or a namespace file, reachable by path would lead to + # the host's processes (/proc//root) or namespaces. + extra = [f"{m['point']} ({m['fstype']})" for m in vis.values() + if (m["fstype"] == "proc" and m["point"] != "/proc") or m["fstype"] == "nsfs"] + f.append((not extra, "no other proc or nsfs mount is reachable" + (f" ({' '.join(extra)})" if extra else ""))) + m = vis.get("/sys/fs/cgroup") + f.append((bool(m) and m["fstype"] == "tmpfs" and "ro" in m["opts"], "/sys/fs/cgroup is masked by a read-only tmpfs")) + if work: + f.append((not left, f"every other mount is read-only ({len(vis)} reachable mounts" + + (f"; writable: {' '.join(left)}" if left else "") + ")")) + return f + + +def eval_stdio(kinds): + """kinds: (fd, kind) of fds 0 to 2 (fd_kind). Only a pipe or /dev/null: + a socket could reach a host service without a connect, a terminal or a + file is the caller's.""" + return [(len(kinds) == 3 and all(k in ("pipe", "/dev/null") for _, k in kinds), + "the standard descriptors are pipes or /dev/null (" + ", ".join(f"{fd}: {k}" for fd, k in kinds) + ")")] + + +def eval_label(own, others, name): + """Findings on AppArmor confinement (G3). own: this process's label or + the OSError reading it; others: (pid, its status or OSError, its label + or OSError) of every other process of the namespace. Every process but + pid 1 (the root shell of the isolation) must carry exactly NAME in + enforce mode: a stacked, complain-mode or other label is refused.""" + want = f"{name} (enforce)" + f = [(bool(NAME_RE.fullmatch(name)), f"the profile name is known ({name or 'NO_DOCKER_PROFILE unset'})")] + mine = own.rstrip("\n") if isinstance(own, str) else f"unreadable ({own})" + f.append((mine == want, f"this process is confined by exactly {want} ({mine})")) + bad = [] + for pid, st, label in others: + if isinstance(st, OSError) or isinstance(label, OSError): + err = st if isinstance(st, OSError) else label + if not gone(err): + bad.append(f"{pid}:unreadable({errno.errorcode.get(err.errno, err.errno)})") + continue + if pid == "1" and st.get("Uid", "").split()[:1] == ["0"] and st.get("Name") in ("sh", "dash"): + continue + if label.rstrip("\n") != want: + bad.append(f"{pid}:{st.get('Name', '?')}:{label.rstrip()}") + f.append((not bad, f"every other process but the root pid 1 is confined by exactly {want}" + + (f" (not: {' '.join(bad)})" if bad else ""))) + return f + + +def eval_fds(o): + """Findings on the descriptor table of the process that becomes the + command (G8). o: {"pid", "listed_pid", "fds": [(fd, fd_kind)]} from a + listing of /proc/self/fd. Exactly fd 0 /dev/null and fds 1 and 2 pipes; + an fd 3 or above listed but closed (EBADF) is the listing's own.""" + kinds = dict(o["fds"]) + std = [(0, "/dev/null"), (1, "pipe"), (2, "pipe")] + f = [(o["listed_pid"] == o["pid"], f"the descriptors listed are this process's (pid {o['pid']}, " + f"listed {o['listed_pid']})")] + f.append((all(kinds.get(fd) == want for fd, want in std), + "fd 0 is /dev/null and fds 1 and 2 are pipes (" + ", ".join(f"{fd}: {kinds.get(fd, 'closed')}" + for fd, _ in std) + ")")) + extra = [f"{fd}: {k}" for fd, k in sorted(kinds.items()) if fd >= 3 and k != "closed"] + f.append((not extra, "no other descriptor is open" + (f" ({', '.join(extra)})" if extra else ""))) + return f + + +def eval_fence(o, uid): + """Findings on the fence (G4) and the twin (G5). o: {"stream": {...}, + "dgram": {...}, "twin": result}, each fence entry {"sock", "uid", "dac", + "result"}. A host-bound socket of our own, which file permissions let + us reach, must be refused with EACCES exactly (only AppArmor gives + that); a socket bound inside must connect.""" + f = [] + for kind, verb in (("stream", "connect"), ("dgram", "sendto")): + e = o.get(kind) or {} + f.append((e.get("sock") is True and e.get("uid") == uid, + f"the {kind} fence is a socket of ours ({e.get('sock')}, uid {e.get('uid')})")) + f.append((e.get("dac") is True, f"file permissions allow the {kind} fence ({e.get('dac')})")) + f.append((e.get("result") == "EACCES", f"{verb} to the {kind} fence is refused with EACCES ({e.get('result')})")) + f.append((o.get("twin") == "ok", f"a socket bound inside the isolation connects ({o.get('twin')})")) + return f + + +def eval_root_fds(before, after, stdin): + """(proceed, closed, findings) of --root-exec's descriptor sanitation. + before, after: (fd, fd_kind) listed before and after the close; stdin: + fd_kind of the new fd 0.""" + closed = [fd for fd, k in before if fd >= 3 and k != "closed"] + left = [f"{fd}: {k}" for fd, k in after if fd >= 3 and k != "closed"] + f = [(not left, "no descriptor above 2 is left" + (f" ({', '.join(left)})" if left else "")), + (stdin == "/dev/null", f"stdin is /dev/null opened inside ({stdin})")] + return all(p for p, _ in f), closed, f + + +def eval_ns_record(text, own_ino): + """Findings on CTLDIR/ns (G2): the record names this pid namespace.""" + m = NS_RE.fullmatch(text) if isinstance(text, str) else None + rec = int(m.group(1)) if m else None + return [(rec is not None and rec == own_ino, + f"the namespace record names this pid namespace ({rec if m else text}; ours {own_ino})")] + + +def evaluate_linux(o): + """Every finding of the mechanism check, from the observations o.""" + f = eval_creds(o["status"], o["group"]) + for ns in ("net", "mnt", "pid", "ipc"): + own, host = o["ns"][ns], o["host_ns"].get(ns, "") + f.append((bool(host) and own != host, f"private {ns} namespace ({own}; caller {host or 'unknown'})")) + f += eval_pidns(o["status"].get("NSpid", ""), o["pid"], o["ns"]["pid"], o["host_ns_raw"], o["proc_self"]) + f += eval_processes(o["processes"], o["uid"]) + f.append((o["sid_ours"], f"a session of our own (session {o['sid']})")) + f.append((o["tty"] != "opened", f"no controlling terminal ({o['tty']})")) + f += eval_stdio(o["stdio"]) + f.append((o["interfaces"] == ["lo"], f"network namespace has only loopback ({' '.join(o['interfaces'])})")) + f.append((bool(o["work"]), f"the run's directory is known ({o['work'] or 'NO_DOCKER_RUN_DIR unset'})")) + f += eval_mounts(o["mounts"], o["host_devs"], o["devs"], o["work"], o["writable"]) + f.append((o["cwd_reachable"] is True, f"the working directory is reachable by its path ({o['cwd']}" + + ("" if o["cwd_reachable"] in (True, False) else f": {o['cwd_reachable']}") + ")")) + return f + + +def fd_kind(fd): + """What an fd is: "pipe", "/dev/null", "terminal", "character device", + "file", "directory", "unix socket", "socket" or "other"; "closed" if + fstat says EBADF; "unreadable ()" for any other error.""" + try: + st = os.fstat(fd) + except OSError as e: + if e.errno == errno.EBADF: + return "closed" + return f"unreadable ({errno.errorcode.get(e.errno, e.errno)})" + mode = st.st_mode + if stat.S_ISFIFO(mode): + return "pipe" + if stat.S_ISCHR(mode): + if st.st_rdev == os.makedev(1, 3): + return "/dev/null" + return "terminal" if os.isatty(fd) else "character device" + if stat.S_ISREG(mode): + return "file" + if stat.S_ISDIR(mode): + return "directory" + if stat.S_ISSOCK(mode): + s = socket.socket(fileno=os.dup(fd)) + try: + return "unix socket" if s.family == socket.AF_UNIX else "socket" + finally: + s.close() + return "other" + + +def list_fds(): + """(fd, fd_kind) of every entry of /proc/self/fd (the listing's own fd + shows as closed), with the pid /proc/self named at the time.""" + listed_pid = int(os.readlink("/proc/self")) + return listed_pid, [(fd, fd_kind(fd)) for fd in sorted(int(x) for x in os.listdir("/proc/self/fd"))] + + +def observe_fds(): + listed_pid, fds = list_fds() + return {"pid": os.getpid(), "listed_pid": listed_pid, "fds": fds} + + +def read_label(pid): + with open(f"/proc/{pid}/attr/apparmor/current") as f: + return f.read() + + +def observe_labels(): + """(this process's label, (pid, status, label) of every other process).""" + try: + own = read_label("self") + except OSError as e: + own = e + others = [] + me = str(os.getpid()) + for d in os.listdir("/proc"): + if d.isdigit() and d != me: + try: + st = status(d) + except OSError as e: + others.append((d, e, e)) + continue + try: + others.append((d, st, read_label(d))) + except OSError as e: + others.append((d, st, e)) + return own, others + + +def observe_linux(): + """What the mechanism check looks at, read once (no write, no connect).""" + st = status() + try: + group = grp.getgrgid(os.getgid()).gr_name + except KeyError: + group = "no group name" + processes = [] + for d in os.listdir("/proc"): + if d.isdigit(): + try: + processes.append((d, status(d))) + except OSError as e: + processes.append((d, e)) + sid = os.getsid(0) + sid_ours = False + if sid > 1: + try: + sid_ours = all(u == str(os.getuid()) for u in status(sid).get("Uid", "").split()) + except OSError: + pass + try: + os.close(os.open("/dev/tty", os.O_RDWR)) + tty = "opened" + except OSError as e: + tty = errno.errorcode.get(e.errno, str(e.errno)) + stdio = [(fd, fd_kind(fd)) for fd in (0, 1, 2)] + work = os.environ.get("NO_DOCKER_RUN_DIR", "") + vis = {m["point"]: m for m in visible_mounts()} + devs = {} + for path, _ in PRIVATE: + try: + devs[path] = str(os.stat(path).st_dev) + except OSError: + devs[path] = "?" + try: + cwd_reachable = mount_id_of(".") == mount_id_of(os.getcwd()) + except OSError as e: + cwd_reachable = str(e) + return { + "status": st, "group": group, "uid": os.getuid(), "pid": os.getpid(), "proc_self": os.readlink("/proc/self"), + "ns": {ns: os.readlink(f"/proc/self/ns/{ns}") for ns in ("net", "mnt", "pid", "ipc")}, + "host_ns": env_pairs("NO_DOCKER_HOST_NS"), "host_ns_raw": os.environ.get("NO_DOCKER_HOST_NS", ""), + "processes": processes, "sid": sid, "sid_ours": sid_ours, "tty": tty, "stdio": stdio, + "interfaces": sorted(n for _, n in socket.if_nameindex()), "work": work, "mounts": vis, + "host_devs": env_pairs("NO_DOCKER_HOST_DEVS"), "devs": devs, + "writable": writable_mounts(work) if work else None, "cwd": os.getcwd(), "cwd_reachable": cwd_reachable, + } + + +def gated(observe, evaluate, after, say): + """Report every finding, and only if there are findings and all passed + run after (the part that writes and connects). Any error before that is + a failure.""" + try: + findings = evaluate(observe()) + except Exception as e: + say(False, f"the isolation check failed: {type(e).__name__}: {e}") + return False + if not findings: + say(False, "the isolation check found nothing to check") + return False + good = True + for passed, text in findings: + say(passed, text) + good = good and passed + if not good: + return False + after() + return True + + +def after_gate_linux(): + """Isolated: writes must fail, no cgroup file is writable and no socket + is in the private directories.""" + protected = [p for p in os.environ.get("NO_DOCKER_PROTECTED", "").split(":") if p] + report(bool(protected), "the protected directories are known (NO_DOCKER_PROTECTED)") + for p in protected: + e = create_error(p) + report(e in (errno.EROFS, errno.EACCES), f"writes to {p} fail ({errno.errorcode.get(e, e) if e else 'written'})") + writable = [] + try: + for dirpath, dirnames, filenames in os.walk("/sys/fs/cgroup"): + for n in dirnames + filenames: + if os.access(os.path.join(dirpath, n), os.W_OK): + writable.append(os.path.join(dirpath, n)) + writable += [p for p in ("/sys/fs/cgroup/cgroup.kill", "/sys/fs/cgroup/cgroup.procs") if os.access(p, os.W_OK)] + except OSError as e: + writable.append(str(e)) + report(not writable, f"no cgroup file is writable ({' '.join(writable) or 'none'})") + try: + found = walk_sockets(["/run", "/tmp", "/var/tmp", "/dev/shm"], limit=20000) + report(not found, f"no socket in the private directories ({' '.join(found) or 'none'})") + except WalkLimit as e: + report(False, f"no socket in the private directories ({e})") + + +def check_mechanism_linux(): + """--check, and a nested run (a suite re-executing itself inside the + isolation): the rules of this process (G1), its own confinement (G3) + and, only if they pass, the after-gate checks (G7); the engine check + follows in main. The descriptor table is reported, not gating: the + first-level gate checked it before the command started.""" + print(f"no-docker: info uids {status().get('Uid', '?')}; gids {status().get('Gid', '?')}") + for passed, text in eval_fds(observe_fds()): + print(f"no-docker: info {'' if passed else 'not: '}{text}") + + def observe(): + o = observe_linux() + try: + o["label"] = read_label("self") + except OSError as e: + o["label"] = e + return o + + gated(observe, lambda o: evaluate_linux(o) + eval_label(o["label"], [], os.environ.get("NO_DOCKER_PROFILE", "")), + after_gate_linux, report) + + +def dac_allows(path): + """True if file permissions alone let this process connect to the + socket at path: write on it and search on every directory above it, + as the effective ids (faccessat AT_EACCESS; AppArmor does not mediate + access(2), so this proves a refusal comes from the profile).""" + try: + if not os.access(path, os.W_OK, effective_ids=True): + return False + d = os.path.dirname(path) + while True: + if not os.access(d, os.X_OK, effective_ids=True): + return False + if d == "/": + return True + d = os.path.dirname(d) + except OSError as e: + return f"unreadable ({errno.errorcode.get(e.errno, e.errno)})" + + +def fence_check(fencedir): + """G4 and G6: the two sockets no-docker.sh bound outside, in fencedir, + looked at, connected and sent to, then closed.""" + o = {} + for kind, stype in (("stream", socket.SOCK_STREAM), ("dgram", socket.SOCK_DGRAM)): + path = os.path.join(fencedir, kind) + e = {} + try: + st = os.lstat(path) + e["sock"], e["uid"] = stat.S_ISSOCK(st.st_mode), st.st_uid + except OSError as x: + e["sock"] = f"unreadable ({errno.errorcode.get(x.errno, x.errno)})" + e["dac"] = dac_allows(path) + s = socket.socket(socket.AF_UNIX, stype) + s.settimeout(2) + try: + if kind == "stream": + s.connect(path) + else: + s.sendto(b"no-docker fence", path) + e["result"] = "ok" + except socket.timeout: + e["result"] = "timeout" + except OSError as x: + e["result"] = errno.errorcode.get(x.errno, str(x.errno)) + finally: + s.close() + o[kind] = e + return o + + +def twin_check(): + """G5 and G6: a stream socket bound inside the isolation (in its private + /tmp) accepts a connection; both are closed and the path removed.""" + path = f"/tmp/.no-docker-twin-{os.getpid()}-{os.urandom(8).hex()}" + srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + cli = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + srv.bind(path) + srv.listen(1) + cli.settimeout(2) + cli.connect(path) + return "ok" + except socket.timeout: + return "timeout" + except OSError as e: + return errno.errorcode.get(e.errno, str(e.errno)) + finally: + cli.close() + srv.close() + try: + os.unlink(path) + except FileNotFoundError: + pass + + +def observe_gate(ctldir, fencedir): + """observe_linux, plus what G2 and G3 look at.""" + o = observe_linux() + o["labels"] = observe_labels() + try: + with open(os.path.join(ctldir, "ns")) as f: + o["ns_text"] = f.read(4096) + except OSError as e: + o["ns_text"] = f"unreadable ({e})" + own = re.fullmatch(r"pid:\[([0-9]+)\]", o["ns"]["pid"]) + o["own_pidns"] = int(own.group(1)) if own else None + o["ro"] = {} + for d in (ctldir, fencedir): + try: + o["ro"][d] = bool(os.statvfs(d).f_flag & os.ST_RDONLY) + except OSError as e: + o["ro"][d] = f"unreadable ({e})" + return o + + +def evaluate_gate(o, name, ctldir, fencedir): + """G1, G2 and G3.""" + f = evaluate_linux(o) + f += eval_ns_record(o["ns_text"], o["own_pidns"]) + for d, what in ((ctldir, "control"), (fencedir, "fence")): + f.append((bool(d) and o["ro"].get(d) is True, f"the {what} directory is read-only inside ({d or 'unknown'}: " + f"{o['ro'].get(d)})")) + f += eval_label(o["labels"][0], o["labels"][1], name) + return f + + +def gate_exec(cmd): + """--gate-exec -- CMD: run as the command will, confined, by --root-exec. + Check G1 to G8, report every finding on fd 2, and only if all passed + reset the signal dispositions, record the start and exec CMD in this + same process, so what was checked is what runs.""" + sys.stdout = sys.stderr # fd 1 is the command's + began = time.monotonic() + name = os.environ.get("NO_DOCKER_PROFILE", "") + ctldir, fencedir = os.environ.get("NO_DOCKER_CTL", ""), os.environ.get("NO_DOCKER_FENCE", "") + print(f"no-docker: info uids {status().get('Uid', '?')}; gids {status().get('Gid', '?')}") + passed = gated(lambda: observe_gate(ctldir, fencedir), lambda o: evaluate_gate(o, name, ctldir, fencedir), + lambda: None, report) + fence = fence_check(fencedir) if fencedir else {} + fence["twin"] = twin_check() + for p, t in eval_fence(fence, os.getuid()): + report(p, t) + if passed and ok: + after_gate_linux() + engine_check() + else: + report(False, "the after-gate checks did not run (an earlier check failed)") + # G8, last: nothing below opens a descriptor that the command inherits. + for p, t in eval_fds(observe_fds()): + report(p, t) + if not ok: + print("no-docker: refused: the isolation could not be proven; the command was not run") + return EX_REFUSED + try: + for sig in signal.valid_signals(): + if sig not in (signal.SIGKILL, signal.SIGSTOP): + signal.signal(sig, signal.SIG_DFL) + signal.pthread_sigmask(signal.SIG_SETMASK, []) + started = os.environ.pop("NO_DOCKER_STARTED") + os.close(os.open(started, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)) + except (OSError, ValueError, KeyError) as e: + print(f"no-docker: refused: cannot start the command ({type(e).__name__}: {e}); the command was not run") + return EX_REFUSED + for v in ("NO_DOCKER_CTL", "NO_DOCKER_FENCE"): + os.environ.pop(v, None) + print(f"no-docker: info the gate passed in {time.monotonic() - began:.2f}s (pid {os.getpid()})") + sys.stderr.flush() + try: + os.execvp(cmd[0], cmd) + except OSError as e: + print(f"no-docker: cannot run {cmd[0]}: {e.strerror}") + return 127 if e.errno == errno.ENOENT else 126 + + +def parse_root_exec(args): + """(uid, gid, name, env, cmd) of `UID GID NAME -- ENV... -- CMD...`; + ValueError for anything else.""" + if len(args) < 6 or args[3] != "--" or "--" not in args[4:]: + raise ValueError("expected UID GID NAME -- NAME=VALUE... -- COMMAND...") + uid, gid, name = args[:3] + if not (uid.isdigit() and gid.isdigit() and int(uid) > 0 and int(gid) > 0): + raise ValueError(f"not an unprivileged uid and gid: {uid} {gid}") + if not NAME_RE.fullmatch(name): + raise ValueError(f"not a profile name: {name!r}") + rest = args[4:] + i = rest.index("--") + env, cmd = rest[:i], rest[i + 1:] + if not cmd or not all(ENV_RE.fullmatch(e) for e in env): + raise ValueError("a malformed environment entry or no command") + return uid, gid, name, env, cmd + + +def root_exec(args): + """--root-exec UID GID NAME -- ENV... -- CMD...: as root inside the + namespaces (a child of pid 1): close every descriptor above 2 and show + that none is left, make fd 0 /dev/null opened here, then exec the chain + that drops the credentials, attaches NAME and runs the gate.""" + try: + uid, gid, name, env, cmd = parse_root_exec(args) + except ValueError as e: + print(f"no-docker: refused: --root-exec: {e}", file=sys.stderr) + return EX_REFUSED + _, before = list_fds() + hard = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + # No descriptor can be above the kernel's nr_open limit (at most 2^30). + os.closerange(3, hard if 0 <= hard < 1 << 30 else 1 << 30) + _, after = list_fds() + fd = os.open("/dev/null", os.O_RDONLY) + if fd != 0: + os.dup2(fd, 0, inheritable=True) + os.close(fd) + else: + os.set_inheritable(0, True) + stdin = fd_kind(0) + proceed, closed, findings = eval_root_fds(before, after, stdin) + remaining = [str(f) for f, k in list_fds()[1] if f < 3 and k != "closed"] + print(f"no-docker: closed: {' '.join(map(str, closed)) or '(none)'}; remaining: {' '.join(remaining)}; " + f"stdin {'char 1:3' if stdin == '/dev/null' else stdin}", file=sys.stderr) + for p, t in findings: + if not p: + print(f"no-docker: FAIL {t}", file=sys.stderr) + if not proceed: + print("no-docker: refused: the descriptors could not be cleaned; the command was not run", file=sys.stderr) + return EX_REFUSED + chain = [SETPRIV, "--reuid", uid, "--regid", gid, "--clear-groups", "--no-new-privs", "--inh-caps=-all", + "--bounding-set=-all", "--", AA_EXEC, "-p", name, "--", SETSID, "--wait", ENV, "-i", *env, + PYTHON, "-I", os.path.abspath(__file__), "--gate-exec", "--", *cmd] + sys.stderr.flush() + try: + os.execv(SETPRIV, chain) + except OSError as e: + print(f"no-docker: refused: cannot run {SETPRIV}: {e.strerror}", file=sys.stderr) + return EX_REFUSED + + +def listening_ports(strict=False): + """(host, port) of listening TCP ports. + + strict (macOS): raise instead of returning a list that may be + incomplete. The list is accepted, even empty, only if netstat succeeded, + printed its column header and every LISTEN row has a parsable local + address. Every address counts (loopback, all, link-local or another): + no-docker.sh holds the port number.""" + ports = set() + if sys.platform == "darwin": + try: + run = subprocess.run(["netstat", "-an", "-p", "tcp"], capture_output=True, text=True, timeout=10) + if run.returncode != 0: + raise OSError(f"netstat exited with {run.returncode}") + out = run.stdout + if strict and not any(line.split()[:4] == ["Proto", "Recv-Q", "Send-Q", "Local"] + for line in out.splitlines()): + raise OSError("netstat printed no column header") + except (OSError, subprocess.SubprocessError): + if strict: + raise + out = "" + for line in out.splitlines(): + f = line.split() + if not f or f[-1] != "LISTEN": + continue + host, _, port = f[3].rpartition(".") if len(f) >= 6 else ("", "", "") + if not host or not port.isdigit(): + if strict: + raise OSError(f"unparsable netstat LISTEN row: {line.strip()}") + continue + ports.add(("::1" if ":" in host else "127.0.0.1", int(port))) + else: + for name in ("/proc/net/tcp", "/proc/net/tcp6"): + try: + with open(name) as f: + next(f) + for line in f: + local, state = line.split()[1], line.split()[3] + if state == "0A": + ports.add(("::1" if name.endswith("6") else "127.0.0.1", int(local.rsplit(":", 1)[1], 16))) + except OSError: + pass + return ports + + +def endpoints(): + home = os.path.expanduser("~") + uid = os.getuid() + unix = { + "/var/run/docker.sock", + "/run/docker.sock", + "/run/containerd/containerd.sock", + "/run/podman/podman.sock", + f"/run/user/{uid}/docker.sock", + f"/run/user/{uid}/podman/podman.sock", + f"{home}/.docker/run/docker.sock", + f"{home}/.docker/desktop/docker.sock", + f"{home}/.colima/default/docker.sock", + f"{home}/.orbstack/run/docker.sock", + f"{home}/.rd/docker.sock", + f"{home}/Library/Containers/com.docker.docker/Data/docker.raw.sock", + f"{home}/Library/Containers/com.docker.docker/Data/docker-cli.sock", + } + tcp = {("127.0.0.1", 2375), ("127.0.0.1", 2376), ("::1", 2375), ("::1", 2376)} + tcp.update(listening_ports()) + hosts = [os.environ.get("NO_DOCKER_ORIGINAL_HOST", ""), os.environ.get("DOCKER_HOST", "")] + for meta in glob.glob(f"{home}/.docker/contexts/meta/*/meta.json"): + try: + with open(meta) as f: + hosts.append(json.load(f)["Endpoints"]["docker"]["Host"]) + except (OSError, ValueError, KeyError, TypeError): + pass + for h in hosts: + if h.startswith("unix://"): + unix.add(h[len("unix://"):]) + elif h.startswith("tcp://"): + hostport = h[len("tcp://"):].split("/", 1)[0] + host, _, port = hostport.rpartition(":") + if host and port.isdigit(): + tcp.add((host.strip("[]"), int(port))) + roots = ["/run", "/var/run", "/tmp", os.environ.get("TMPDIR", ""), f"{home}/.docker", + f"{home}/Library/Containers/com.docker.docker/Data"] + try: + unix.update(walk_sockets(roots)) + except WalkLimit as e: + if sys.platform != "darwin": + raise + # The sandbox denies every Unix socket connect (checked first); the + # walk is a second line only. Said, not hidden. + print(f"no-docker: info socket walk incomplete, {e}; Unix sockets are denied by the sandbox") + return sorted(unix), sorted(tcp) + + +def main(): + if sys.argv[1:] == ["--listening"]: + # For no-docker.sh, before the isolation: the ports to deny inside it. + try: + ports = listening_ports(strict=True) + except (OSError, subprocess.SubprocessError) as e: + print(f"no-docker: cannot list the listening ports: {e}", file=sys.stderr) + return 1 + print("\n".join(sorted({str(port) for _, port in ports}, key=int))) + return 0 + if sys.platform == "darwin" and (sys.argv[1:2] == ["--markers"] and len(sys.argv) == 4 + or sys.argv[1:2] in (["--orphans"], ["--end"]) and len(sys.argv) == 3): + try: + if sys.argv[1] == "--markers": + return darwin_markers(sys.argv[2], sys.argv[3]) + if sys.argv[1] == "--orphans": + return darwin_orphans(sys.argv[2]) + sbx = Sandboxes(sys.argv[2]) + try: + alive = owner_alive(sbx, sys.argv[2]) + except FileNotFoundError: + alive = False + if alive: + raise OSError(f"the run of {sys.argv[2]} is still in progress (its owner runs); not ending it") + try: + gone_owner = owner_record(sys.argv[2])[1] + except FileNotFoundError: + gone_owner = None + n = sweep(sbx, 10.0, kill=True, owner=gone_owner)[0] + shutil.rmtree(sys.argv[2], ignore_errors=True) + print(f"no-docker: ended {n} processes; none is left", file=sys.stderr) + return 0 + except OSError as e: + print(f"no-docker: cannot check the processes of the isolation: {e}", file=sys.stderr) + return 1 + if sys.platform == "darwin" and sys.argv[1:2] == ["--supervise"] and len(sys.argv) > 6 and sys.argv[5] == "--": + return darwin_supervise(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[6:]) + if sys.argv[1:2] == ["--root-exec"] and sys.platform.startswith("linux"): + try: + return root_exec(sys.argv[2:]) + except Exception as e: # nothing was run + print(f"no-docker: refused: --root-exec failed ({type(e).__name__}: {e})", file=sys.stderr) + return EX_REFUSED + if sys.argv[1:3] == ["--gate-exec", "--"] and len(sys.argv) > 3 and sys.platform.startswith("linux"): + try: + return gate_exec(sys.argv[3:]) + except Exception as e: # any doubt: the command does not run + print(f"no-docker: refused: the gate failed ({type(e).__name__}: {e}); the command was not run", + file=sys.stderr) + return EX_REFUSED + if sys.argv[1:2] == ["--seal"] and len(sys.argv) == 3: + try: + return seal(sys.argv[2]) + except OSError as e: + print(f"no-docker: cannot make the mounts read-only: {e}", file=sys.stderr) + return 1 + if sys.argv[1:]: + print(f"no-docker: unknown arguments {' '.join(sys.argv[1:])}", file=sys.stderr) + return 2 + try: + if sys.platform == "darwin": + check_mechanism_darwin() + elif sys.platform.startswith("linux"): + check_mechanism_linux() + else: + report(False, f"no isolation mechanism for platform {sys.platform}") + except OSError as e: + report(False, f"the isolation check failed: {e}") + if not ok: + # Not isolated: do not send anything to an engine that may answer. + return 1 + engine_check() + return 0 if ok else 1 + + +def engine_check(): + """No container engine answers on any endpoint endpoints() finds. Run + only once the mechanism checks passed.""" + try: + unix, tcp = endpoints() + except WalkLimit as e: + report(False, f"every socket of the walked directories was checked ({e})") + return + + def ask(target): + kind, addr = target + try: + if kind == "unix": + a = engine_answers(socket.AF_UNIX, addr) + else: + a = engine_answers(socket.AF_INET6 if ":" in addr[0] else socket.AF_INET, addr, 1) + except OSError: + a = None + if not a: + return None + return f"unix://{addr} ({a})" if kind == "unix" else f"tcp://{addr[0]}:{addr[1]} ({a})" + + targets = [("unix", p) for p in unix] + [("tcp", t) for t in tcp] + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: + found = [r for r in pool.map(ask, targets) if r] + report(not found, f"no container engine answers on {len(unix)} socket and {len(tcp)} TCP endpoints" + + (": " + ", ".join(found) if found else "")) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/no-docker.sh b/scripts/ci/no-docker.sh new file mode 100755 index 0000000..ec9c9ca --- /dev/null +++ b/scripts/ci/no-docker.sh @@ -0,0 +1,903 @@ +#!/usr/bin/env bash +# +# Run a command where no container engine can be reached and where it cannot +# change what runs or is read outside it later, so a test suite, and any +# modified or mutated copy of the code it runs, cannot touch the Docker +# daemon of the machine (its images, containers, volumes, networks or build +# cache) or the user's environment. docs/dev-environment.md, "Isolation from +# Docker", lists every property, how it is verified and what is left. +# +# Usage: no-docker.sh [--work DIR] COMMAND [ARG...] +# run COMMAND isolated; its status +# no-docker.sh --check exit 0 only if this process is already +# isolated +# no-docker.sh --negative-control COMMAND [ARG...] +# Linux on GitHub-hosted runners, the test +# suite's control only: launch under a +# profile that does not deny host sockets +# (the negative template), so the gate must +# refuse; COMMAND is replaced by +# /usr/bin/true and never runs, and the +# status is always 125 +# +# --work DIR: use DIR, a fresh, empty directory of the caller (mode 0700, not +# a symbolic link and without one in its path), as the only writable +# directory of the command instead of a temporary one, and keep it +# afterwards. On macOS DIR must be under $TMPDIR or /private/var/tmp, on +# Linux under /var/tmp. Whatever the command leaves there is untrusted data: +# a later reader must not follow symbolic links, must accept regular files +# only, must limit their size and must not run them. +# +# Supported: macOS developer machines (sandbox-exec) and Linux on +# GitHub-hosted Actions runners (sudo unshare). Everything else, including +# Linux elsewhere and uid 0, is refused. +# macOS sandbox-exec: no connect to a Unix socket, no TCP beyond +# loopback, to loopback ports 22, 2375 and 2376 or to a loopback +# port that was listening before the command started; file writes +# only in the run's own directory (the command's $TMPDIR is in it) +# and the /dev nodes a shell needs, nothing in /tmp (so no bash 3.2 +# here-document inside); no signal to and no debugging of a +# process outside the sandbox; no preference write, TIOCSTI, open, +# osascript, launchctl, ssh, launchd job, Launch Services, Apple +# Events, login item or background task registration and no lookup +# of Docker's own services. When the command ends, or no-docker.sh +# gets SIGINT, SIGTERM, SIGHUP or SIGQUIT, the processes left in +# the sandbox (found by a marker of the run) are killed; the run +# is reported done only when it is shown that none of them runs +# any more, otherwise it fails (125). A run that ended so, or +# without its supervisor (SIGKILL), keeps new runs from starting +# until that is shown. $TMPDIR must be the per-user directory +# under /private/var/folders. +# Linux sudo unshare and AppArmor: new network (loopback only), mount, pid +# and IPC namespaces; empty tmpfs over /run, /var/run, /tmp, +# /var/tmp, /dev/shm and /sys/fs/cgroup, a fresh /dev/mqueue, every +# other mount read-only; the command runs as the calling user without +# supplementary groups, capabilities or the means to regain them +# (no_new_privs), in a session of its own, with an allowlisted +# environment, under a root pid 1 that waits for it, confined by an +# AppArmor profile loaded for this launch alone +# (scripts/ci/no-docker/apparmor.profile, no attach_disconnected): +# the kernel refuses every Unix socket bound outside the namespace +# (EACCES), whatever its path. Before the command starts it is shown +# on two sockets of this launch bound outside (the fence) that the +# refusal is in effect. stdin is /dev/null; stdout and stderr reach +# the caller through pipes of this script, and the caller's must +# not be a socket or a terminal. When the command ends, when +# no-docker.sh itself gets SIGINT, SIGTERM or SIGHUP (such as Ctrl-C, +# or a caller forwarding the signal) or when unshare dies, every +# process left inside is killed; the profile is removed only when +# the kernel shows no process can carry it any more, otherwise it +# stays loaded, the run fails (125) and says so. A cancelled or +# timed-out GitHub step does not signal no-docker.sh: its namespace +# keeps running, confined, read-only and without network, and the +# profile stays loaded, until the VM is discarded (an accepted +# residual). docs/dev-environment.md, "Linux", has the details. +# In both, the variables that select an engine are removed and DOCKER_CONFIG +# is an empty directory. Before COMMAND runs, no-docker-probe.py proves from +# inside that each mechanism is in effect and that no engine answers on any +# endpoint it can find. If isolation cannot be applied or proven, COMMAND +# does not run and the exit status is 125; so is the status if the +# isolation tool returns without the command having started inside it, and +# on macOS if the processes the command left could not all be ended. + +set -u + +here=$(cd -P "$(dirname "$0")" && pwd -P) +probe="$here/no-docker-probe.py" +aa="$here/no-docker-aa.py" +EX_REFUSED=125 + +refuse() { + printf 'no-docker: refused: %s; the command was not run\n' "$1" >&2 + exit "$EX_REFUSED" +} + +if [ "${1-}" = --check ]; then + exec python3 -I "$probe" +fi + +if [ "${1-}" = --inner ]; then + # Inside the macOS sandbox: prove it, then run the command. (On Linux + # no-docker-probe.py --gate-exec does this.) + [ "$(uname -s)" = Darwin ] || refuse "--inner is used on macOS only" + shift + python3 -I "$probe" >&2 || refuse "the isolation could not be proven" + : >"$NO_DOCKER_STARTED" || refuse "cannot record the start" + unset NO_DOCKER_STARTED + exec "$@" +fi + +negative="" +if [ "${1-}" = --negative-control ]; then + negative=yes + shift +fi +given_work="" +if [ "${1-}" = --work ]; then + [ $# -ge 2 ] || refuse "--work needs a directory" + given_work=$2 + shift 2 +fi +[ $# -gt 0 ] || refuse "no command given" +if [ -n "${NO_DOCKER_ACTIVE-}" ]; then + # Already isolated (a suite re-executing itself): never nest. + [ -z "$negative" ] || refuse "--negative-control is refused inside the isolation" + python3 -I "$probe" >&2 || refuse "inside no-docker.sh, but the isolation check fails" + [ -z "$given_work" ] || refuse "--work cannot be used inside the isolation" + exec "$@" +fi + +# uid 0 keeps every capability through the isolation; refused first. +if [ "$(id -u)" = 0 ] || [ "$(id -ru)" = 0 ] || [ "${EUID-}" = 0 ] || [ "${UID-}" = 0 ]; then + refuse "running as root (uid 0); run it as an unprivileged user" +fi + +os=$(uname -s) +uid=$(id -u) +[ -z "$negative" ] || [ "$os" = Linux ] || refuse "--negative-control is for Linux on GitHub-hosted runners only" + +# plain PATH: true if PATH has only characters that are safe in a sandbox +# profile and in the shell of the namespace. +plain() { + case $1 in "" | *[!A-Za-z0-9/._-]*) return 1 ;; esac + return 0 +} + +# owner_mode PATH: " " of PATH. +owner_mode() { + if [ "$os" = Darwin ]; then stat -f '%u %Lp' "$1"; else stat -c '%u %a' "$1"; fi +} + +# use_work DIR PARENT...: check the caller's --work DIR and use it as $work. +use_work() { + local d=$1 canon real p inside=no + shift + case $d in /*) ;; *) refuse "--work $d is not an absolute path" ;; esac + plain "$d" || refuse "unexpected character in --work $d" + case $d in */ | *//* | */./* | */../* | */. | */..) refuse "--work $d is not a canonical path" ;; esac + [ -d "$d" ] && [ ! -L "$d" ] || refuse "--work $d is not a directory or is a symbolic link" + canon=$d + if [ "$os" = Darwin ]; then + case $d in /var/* | /tmp/*) canon=/private$d ;; esac + fi + real=$(cd -P "$d" && pwd -P) || refuse "cannot resolve --work $d" + [ "$real" = "$canon" ] || refuse "--work $d has a symbolic link in its path ($real)" + for p in "$@"; do + case $real in "$p"/?*) inside=yes ;; esac + done + [ "$inside" = yes ] || refuse "--work $d is not under $*" + case $real/ in "$home"/*) refuse "--work $d is under \$HOME" ;; esac + case $home/ in "$real"/*) refuse "--work $d contains \$HOME" ;; esac + [ "$(owner_mode "$real")" = "$uid 700" ] || refuse "--work $d is not owned by you with mode 0700" + [ -z "$(ls -A "$real")" ] || refuse "--work $d is not empty" + work=$real +} + +home=$(cd -P "$HOME" && pwd -P) || refuse "cannot resolve \$HOME" +plain "$home" || refuse "unexpected character in \$HOME ($home)" + +case $os in + Darwin) + command -v sandbox-exec >/dev/null || refuse "sandbox-exec not found" + # $TMPDIR must be the per-user temporary directory macOS creates, + # /private/var/folders///T, owned by the user with mode 0700: + # any other directory may hold files that run or are read outside the + # sandbox. The command gets a directory of its own inside it. + t=${TMPDIR-} + [ -n "$t" ] || t=$(getconf DARWIN_USER_TEMP_DIR 2>/dev/null) || refuse "TMPDIR is unset and getconf DARWIN_USER_TEMP_DIR fails" + tmp=$(cd -P "$t" 2>/dev/null && pwd -P) || refuse "cannot resolve the temporary directory $t" + plain "$tmp" || refuse "unexpected character in the temporary directory $tmp" + case $tmp in + /private/var/folders/[!/][!/]/?*/T) ;; + *) refuse "the temporary directory $tmp is not the per-user directory /private/var/folders///T; unset TMPDIR or set it to that directory" ;; + esac + rest=${tmp#/private/var/folders/??/} + case ${rest%/T} in "" | */*) refuse "the temporary directory $tmp is not the per-user directory /private/var/folders///T" ;; esac + [ "$(owner_mode "$tmp")" = "$uid 700" ] || refuse "the temporary directory $tmp is not owned by you with mode 0700" + if [ -n "$given_work" ]; then + use_work "$given_work" "$tmp" /private/var/tmp + else + work=$(mktemp -d "$tmp/no-docker.XXXXXX") || refuse "no temporary directory" + work=$(cd -P "$work" && pwd -P) || refuse "cannot resolve the temporary directory" + trap 'rm -rf "$work"' EXIT + fi + # The sandbox carries a marker of this run (a denied read of a file in + # its marker directory, which is in $tmp, outside everything the + # sandbox may write), by which no-docker-probe.py finds its processes + # from outside: a process cannot leave its sandbox, and every process it + # starts inherits it. The directory records this process as the run's + # owner (the supervisor it becomes). An earlier run that ended without + # its supervisor and may have processes left is ruled out before + # anything is written into the run's directory. + markdir=$(python3 -I "$probe" --markers "$tmp" "$$") || refuse "cannot make the marker of the run" + case $markdir in "$tmp"/no-docker-mark.?*) ;; *) refuse "unexpected marker directory $markdir" ;; esac + plain "$markdir" || refuse "unexpected character in the marker directory $markdir" + if [ -n "$given_work" ]; then + remove=- + trap 'rm -rf "$markdir"' EXIT + else + remove=$work + trap 'rm -rf "$work" "$markdir"' EXIT + fi + python3 -I "$probe" --orphans "$tmp" || refuse "cannot rule out a process left by an earlier isolated run" + ;; + Linux) + # --- Linux: the launch's processes, state and cleanup ------------------------- + # A pid alone may name another process once the one recorded has ended and + # been reaped (bash reaps its children itself), so every process this script + # acts on is identified by its pid and its start time (field 22 of + # /proc//stat), read when it was started. + + # proc_stat PID: set ps_comm, ps_state, ps_ppid and ps_start from + # /proc/PID/stat; false if it cannot be read (the process is gone). + proc_stat() { + local line rest + { IFS= read -r line <"/proc/$1/stat"; } 2>/dev/null || return 1 + ps_comm=${line#*(} + ps_comm=${ps_comm%)*} + rest=${line##*) } + # shellcheck disable=SC2086 # the fields, split on purpose + set -- $rest + [ $# -ge 20 ] || return 1 + ps_state=$1 ps_ppid=$2 ps_start=${20} + } + + # alive PID START: true while the process PID started at START runs (not a + # zombie). + alive() { + [ -n "$1" ] && [ -n "$2" ] && proc_stat "$1" && [ "$ps_state" != Z ] && [ "$ps_start" = "$2" ] + } + + # wait_gone PID START SECONDS: true once the process no longer runs; false + # if it still does after SECONDS. A start time not recorded yet is read now. + wait_gone() { + local pid=$1 start=$2 end + [ -n "$pid" ] || return 0 + if [ -z "$start" ]; then + proc_stat "$pid" || return 0 + start=$ps_start + fi + end=$((${EPOCHREALTIME//[!0-9]/} + $3 * 1000000)) + while alive "$pid" "$start"; do + [ "${EPOCHREALTIME//[!0-9]/}" -lt "$end" ] || return 1 + sleep 0.05 + done + return 0 + } + + # wait_all SECONDS PID:START...: true once none of the processes runs; + # false if one still does after SECONDS (all share the one deadline). + wait_all() { + local end x p s busy + end=$((${EPOCHREALTIME//[!0-9]/} + $1 * 1000000)) + shift + while :; do + busy="" + for x in "$@"; do + p=${x%%:*} s=${x#*:} + if [ -n "$p" ] && [ -z "$s" ]; then + proc_stat "$p" || continue + s=$ps_start + fi + if alive "$p" "$s"; then busy=yes; fi + done + [ -n "$busy" ] || return 0 + [ "${EPOCHREALTIME//[!0-9]/}" -lt "$end" ] || return 1 + sleep 0.05 + done + } + + # wait_file PATH: true once PATH exists (within 5 s). + wait_file() { + local i=0 + while [ ! -e "$1" ]; do + [ "$i" -lt 100 ] || return 1 + sleep 0.05 + i=$((i + 1)) + done + } + + # put_state VALUE: record the launch's state in CTLDIR/state: a new file + # written, synced and renamed over the old one, then read back. The value + # is taken as the state before the write, so a signal during it errs on + # the side of a later state. + put_state() { + local got + state=$1 + printf '%s\n' "$1" >"$ctldir/.state" && sync "$ctldir/.state" && mv -f "$ctldir/.state" "$ctldir/state" && + { IFS= read -r got <"$ctldir/state"; } 2>/dev/null && [ "$got" = "$1" ] + } + + # resolve_pending: a start interrupted between its "&" and the recording of + # its pid is resolved from $!: changed since the marker was set, the fork + # happened and $! is the new process; unchanged, it did not. The pids are + # compared by number: the kept pid is that of a process this shell + # started last, and the same number could come back between two adjacent + # commands only if the kernel's pid space wrapped meanwhile (4194304 + # pids), which is taken as impossible. + resolve_pending() { + if [ -n "$pending" ] && [ "$!" != "$kept" ]; then + case $pending in + stream) fspid=$! ;; + dgram) fdpid=$! ;; + load) lpid=$! ;; + relay-o) opid=$! ;; + relay-e) epid=$! ;; + init) spid=$! ;; + esac + fi + pending="" + } + + # relay_ok PID: true if PID is a child of this shell running cat (the exec + # may take a moment: up to 1 s); sets ps_start. + relay_ok() { + local i=0 + while [ "$i" -lt 50 ]; do + proc_stat "$1" || return 1 + [ "$ps_ppid" = $$ ] || return 1 + [ "$ps_comm" = cat ] && return 0 + sleep 0.02 + i=$((i + 1)) + done + return 1 + } + + # own_fds: list this shell's descriptors into CTLDIR/fds (by a child, so the + # listing's own descriptor is not among them) and set fd_dup to the other + # descriptors of a relay's pipe and fd_bad to every other unexpected one. + own_fds() { + local line fd target ot et + ls -lv "/proc/$$/fd/" >"$ctldir/fds" 2>&1 || return 1 + ot=$(readlink "/proc/$$/fd/$o") && et=$(readlink "/proc/$$/fd/$e") || return 1 + fd_dup="" fd_bad="" + while IFS= read -r line; do + case $line in *" -> "*) ;; *) continue ;; esac + target=${line#* -> } + fd=${line% -> *} + fd=${fd##* } + case $fd in 0 | 1 | 2 | 255 | "$o" | "$e") continue ;; esac + if [ "$target" = "$ot" ] || [ "$target" = "$et" ]; then + fd_dup="$fd_dup $fd" + else + fd_bad="$fd_bad $fd:$target" + fi + done <"$ctldir/fds" + } + + # sudo_may_run: true unless the sudo that starts the namespaces is known to + # have ended (reaped by the wait of L9, or no longer running). + sudo_may_run() { + [ -z "$sudo_reaped" ] || return 1 + if [ -z "$sstart" ]; then + proc_stat "$spid" || return 1 + sstart=$ps_start + fi + alive "$spid" "$sstart" + } + + # close_relay_ends: close every descriptor of this shell on a relay's + # pipe ($o, $e and any copy bash keeps of a process substitution), so + # the relays see the end of their input once the command's copies are + # gone. + close_relay_ends() { + local f fd t ot="" et="" + [ -z "$o" ] || ot=$(readlink "/proc/$$/fd/$o" 2>/dev/null) + [ -z "$e" ] || et=$(readlink "/proc/$$/fd/$e" 2>/dev/null) + for f in "/proc/$$/fd"/*; do + fd=${f##*/} + case $fd in "" | *[!0-9]* | 0 | 1 | 2 | 255) continue ;; esac + t=$(readlink "$f" 2>/dev/null) || continue + if { [ -n "$ot" ] && [ "$t" = "$ot" ]; } || { [ -n "$et" ] && [ "$t" = "$et" ]; }; then + exec {fd}>&- + fi + done + o="" e="" + } + + # secs MICROSECONDS: as seconds with two decimals. + secs() { + printf '%d.%02d' $(($1 / 1000000)) $(($1 % 1000000 / 10000)) + } + + # cleanup (the EXIT trap, once): end what this launch started, each step + # only for what was recorded as started and bounded in time, keep the + # profile loaded unless no-docker-aa.py shows that no process can carry it, + # and exit with the launch's status. + # shellcheck disable=SC2317,SC2329 # run as the EXIT trap + cleanup() { + trap '' INT TERM HUP + local left="" fail="" started=no status t + [ -z "$cleaning" ] || return + cleaning=yes + t=${EPOCHREALTIME//[!0-9]/} + close_relay_ends + resolve_pending + # C1: pid 1 of the namespace, killed by no-docker-aa.py, only while sudo + # may still run; a fork that did not happen is recorded as such. + case $state in + entering | entered*) + if [ -z "$spid" ]; then + put_state not-forked || fail="$fail; the state not-forked could not be recorded" + elif sudo_may_run; then + sudo -n timeout 10 /usr/bin/python3 -I "$aa" kill-init "$$" "$ctldir" || fail="$fail; kill-init failed" + wait_gone "$spid" "$sstart" 10 || left="the namespace did not end within 10s" + fi + ;; + esac + # C2: the relays end once every writer is gone; only then may workflow + # commands resume. + if wait_all 30 "$opid:$ostart" "$epid:$estart"; then + if [ -n "$opid" ]; then + printf 'no-docker: info the relays ended %ss after the command\n' "$(secs $((${EPOCHREALTIME//[!0-9]/} - t)))" >&2 + fi + # On a line of its own even if the command's last line had no newline. + if [ "$stopped" = yes ]; then printf '\n::%s::\n' "$token"; fi + else + fail="$fail; a relay of the command's output did not end (workflow commands stay stopped)" + fi + # C3: the fence listeners, by identity only. + for x in "$fspid:$fsstart" "$fdpid:$fdstart"; do + if alive "${x%%:*}" "${x#*:}"; then kill -TERM "${x%%:*}" 2>/dev/null; fi + done + if ! wait_all 2 "$fspid:$fsstart" "$fdpid:$fdstart"; then + for x in "$fspid:$fsstart" "$fdpid:$fdstart"; do + if alive "${x%%:*}" "${x#*:}"; then kill -KILL "${x%%:*}" 2>/dev/null; fi + done + fi + if [ -n "$fencedir" ] && ! rm -rf "$fencedir"; then + fail="$fail; the fence directory $fencedir could not be removed" + fi + # C4: a load still in flight. + if [ -n "$lpid" ] && [ -z "$load_done" ]; then + wait_gone "$lpid" "$lstart" 20 || left="${left:-the profile load did not end within 20s}" + fi + # C5: the release, only of a profile whose load was started. + if [ -n "$lpid" ] && [ -z "$left" ]; then + local tr=${EPOCHREALTIME//[!0-9]/} + sudo -n timeout 30 /usr/bin/python3 -I "$aa" release "$profile" "$ctldir" || + left="no-docker-aa.py release refused or failed" + t_release=$((${EPOCHREALTIME//[!0-9]/} - tr)) + fi + [ -z "$work" ] || [ ! -e "$work/started" ] || started=yes + # C6 + if [ -n "$left$fail" ]; then + if [ -n "$left" ]; then + sudo -n timeout 10 /usr/bin/python3 -I "$aa" diag "$profile" + printf 'no-docker: refused: profile %s still loaded (%s); cleanup failed; its control directory %s is kept\n' \ + "$profile" "$left" "$ctldir" >&2 + printf '::error title=no-docker::profile %s still loaded (%s); control directory %s\n' "$profile" "$left" "$ctldir" + fi + [ -z "$fail" ] || printf 'no-docker: refused: cleanup failed (%s)\n' "${fail#; }" >&2 + [ -z "$ctldir" ] || printf 'no-docker: the control directory %s is kept\n' "$ctldir" >&2 + elif [ -n "$ctldir" ] && ! rm -rf "$ctldir"; then + fail="the control directory $ctldir could not be removed" + printf 'no-docker: refused: cleanup failed (%s)\n' "$fail" >&2 + fi + # The run directory is removed only once the namespace is shown gone: + # a process left inside could otherwise swap a symbolic link into it + # while it is changed. + if [ "$work_made" = yes ] && [ -z "$left" ]; then + chmod -R u+rwx "$work" 2>/dev/null + if ! rm -rf "$work"; then + fail="${fail:+$fail; }the run directory $work could not be removed" + printf 'no-docker: refused: cleanup failed (the run directory %s could not be removed)\n' "$work" >&2 + fi + elif [ "$work_made" = yes ]; then + printf 'no-docker: the run directory %s is kept\n' "$work" >&2 + fi + # C7: the status. A cleanup failure is 125 even after the command + # succeeded; the negative control is always 125; then a signal; then a + # refusal; then a command that did not start; then the command's own. + if [ -n "$left$fail" ]; then + status=$EX_REFUSED + elif [ -n "$negative" ]; then + status=$EX_REFUSED + elif [ -n "$signalled" ]; then + status=$signalled + elif [ "$normal" != yes ]; then + status=$EX_REFUSED + elif [ "$started" != yes ]; then + printf 'no-docker: refused: the isolation tool returned %s without starting the command\n' "$rc" >&2 + status=$EX_REFUSED + else + status=$rc + fi + if [ -n "$lpid" ]; then + printf '::notice title=no-docker::launch %ss: preflight %ss, fence %ss, profile load %ss, relays %ss, namespace, gate and command %ss, release %ss, cleanup %ss\n' \ + "$(secs $((${EPOCHREALTIME//[!0-9]/} - t_launch)))" "$(secs "$t_preflight")" "$(secs "$t_fence")" "$(secs "$t_load")" \ + "$(secs "$t_relays")" "$(secs "$t_run")" "$(secs "$t_release")" "$(secs $((${EPOCHREALTIME//[!0-9]/} - t)))" >&2 + fi + exit "$status" + } + + # L0: the state of the launch, all empty, and the traps, before any + # resource exists. Every way out (the normal end, a refusal, INT, TERM, + # HUP) goes through cleanup, once, which touches only what was recorded + # as started. "$!" has a value from here on (": &"), so the starts below + # can be told apart from it under set -u. + ctldir="" fencedir="" profile="" load_op="" state="" lpid="" lstart="" load_done="" spid="" sstart="" sudo_reaped="" + o="" e="" opid="" ostart="" epid="" estart="" fspid="" fsstart="" fdpid="" fdstart="" + pending="" kept="" token="" stopped="" signalled="" normal="" cleaning="" work="" work_made="" rc="" + t_launch=${EPOCHREALTIME//[!0-9]/} t_preflight=0 t_load=0 t_fence=0 t_relays=0 t_run=0 t_release=0 + trap '' PIPE + # On Linux a refusal stops taking signals before it exits, so that a + # signal cannot cut short the cleanup its exit runs. + refuse() { + trap '' INT TERM HUP + printf 'no-docker: refused: %s; the command was not run\n' "$1" >&2 + exit "$EX_REFUSED" + } + : & + kept=$! + trap cleanup EXIT + trap 'trap "" INT TERM HUP; signalled=130; exit 130' INT + trap 'trap "" INT TERM HUP; signalled=143; exit 143' TERM + trap 'trap "" INT TERM HUP; signalled=129; exit 129' HUP + # Only GitHub-hosted runners: the VM is discarded after the job, sudo + # needs no password and no rootless engine or user session exists. The + # checks below guard against using this elsewhere by accident; they are + # not a boundary against the caller. + [ "${GITHUB_ACTIONS-}" = true ] && [ "${RUNNER_ENVIRONMENT-}" = github-hosted ] || + refuse "Linux is supported on GitHub-hosted CI runners only (GITHUB_ACTIONS and RUNNER_ENVIRONMENT)" + found=no + pid=$PPID + n=0 + while [ "$pid" -gt 1 ] && [ "$n" -lt 64 ]; do + comm=$(cat "/proc/$pid/comm" 2>/dev/null) || break + if [ "$comm" = Runner.Worker ]; then + exe=$(readlink "/proc/$pid/exe" 2>/dev/null) || break + [ "${exe##*/}" = Runner.Worker ] && found=yes + break + fi + st=$(cat "/proc/$pid/stat" 2>/dev/null) || break + rest=${st##*) } + rest=${rest#* } + pid=${rest%% *} + case $pid in "" | *[!0-9]*) break ;; esac + n=$((n + 1)) + done + [ "$found" = yes ] || refuse "Linux is supported on GitHub-hosted CI runners only (no Runner.Worker ancestor)" + # The new pid namespace is shown to be below the caller's by the + # caller's being the initial one (no-docker-probe.py). + [ "$(readlink /proc/self/ns/pid)" = "pid:[4026531836]" ] || + refuse "Linux is supported on GitHub-hosted CI runners only (not in the initial pid namespace)" + protected=$home + for v in GITHUB_WORKSPACE RUNNER_TEMP; do + d=$(cd -P "${!v:-/nonexistent}" 2>/dev/null && pwd -P) || refuse "Linux is supported on GitHub-hosted CI runners only ($v is not a directory)" + plain "$d" || refuse "unexpected character in $v ($d)" + case $d in "$home"/?*) ;; *) refuse "Linux is supported on GitHub-hosted CI runners only ($v is not under \$HOME)" ;; esac + protected="$protected:$d" + runner_temp=$d + done + # L1: the caller's stdout and stderr receive what the relays copy from + # the command; one that is a socket or a terminal would be a channel out + # of the isolation. stdin is not used: the command gets /dev/null. + for fd in 1 2; do + [ -e "/proc/$$/fd/$fd" ] || refuse "the caller's fd $fd is closed" + if [ -S "/proc/$$/fd/$fd" ] || [ -t "$fd" ]; then + refuse "the caller's fd $fd is a socket or a terminal; give the command a pipe or a file" + fi + done + for tool in sudo timeout unshare setpriv setsid ip mount find stat python3 od sync; do + command -v "$tool" >/dev/null || refuse "$tool not found" + done + for tool in /usr/bin/python3 /usr/bin/setpriv /usr/bin/aa-exec /usr/bin/setsid /usr/bin/env; do + [ -x "$tool" ] || refuse "$tool not found" + done + sudo -n true 2>/dev/null || refuse "creating the namespaces needs passwordless sudo" + t=${EPOCHREALTIME//[!0-9]/} + sudo -n timeout 10 /usr/bin/python3 -I "$aa" preflight || refuse "AppArmor cannot confine the launch (no-docker-aa.py preflight)" + t_preflight=$((${EPOCHREALTIME//[!0-9]/} - t)) + if [ -n "$given_work" ]; then + use_work "$given_work" /var/tmp + else + work=$(mktemp -d /tmp/no-docker.XXXXXX) || refuse "no temporary directory" + work_made=yes + work=$(cd -P "$work" && pwd -P) || refuse "cannot resolve the temporary directory" + fi + cwd=$(pwd -P) || refuse "cannot resolve the working directory" + # What the probe compares inside: the namespaces and the devices of the + # directories that must be private. + host_ns="" + for ns in net mnt pid ipc; do + v=$(readlink "/proc/self/ns/$ns") || refuse "cannot read the $ns namespace" + host_ns="$host_ns $ns=$v" + done + host_devs="" + for d in /tmp /var/tmp /dev/shm /run /dev/mqueue; do + v=$(stat -c %d "$d") || refuse "cannot stat $d" + host_devs="$host_devs $d=$v" + done + # The command keeps the caller's primary group (the probe refuses a + # system group); for the log, next to the group of the engine socket. + gid=$(id -g) + printf 'no-docker: info primary group %s (%s); /var/run/docker.sock group %s\n' "$gid" \ + "$(id -gn 2>/dev/null || echo '?')" "$(stat -Lc '%g (%G)' /var/run/docker.sock 2>/dev/null || echo none)" >&2 + ;; + *) + refuse "no isolation mechanism for $os" + ;; +esac +plain "$work" || refuse "unexpected character in the working directory $work" +mkdir "$work/docker-config" "$work/bin" "$work/tmp" || refuse "cannot prepare $work" + +# The environment of the command. +envargs=() +while IFS= read -r -d '' kv; do + name=${kv%%=*} + if [ "$os" = Linux ]; then + # An allowlist: no session bus, display, agent, runner file-command or + # token variables. + case $name in + PATH | HOME | USER | LOGNAME | SHELL | LANG | LANGUAGE | LC_* | TERM | TZ | CI | DEVENV | DEVENV_* | GUARD | GUARD_* | ADAPTED | HARNESS | FLOWCRM_*) envargs+=("$kv") ;; + esac + else + case $name in + DOCKER_* | CONTAINER_HOST | CONTAINER_CONNECTION | CONTAINERD_* | BUILDKIT_HOST | COMPOSE_* | PODMAN_* | NO_DOCKER_* | TMPDIR) ;; + *) envargs+=("$kv") ;; + esac + fi +done < <(env -0) +envargs+=("DOCKER_CONFIG=$work/docker-config" "TMPDIR=$work/tmp" "NO_DOCKER_ACTIVE=1" + "NO_DOCKER_ORIGINAL_HOST=${DOCKER_HOST-}" "NO_DOCKER_STARTED=$work/started" "NO_DOCKER_RUN_DIR=$work") + +case $os in + Darwin) + # A copy of ps loses its setuid bit; it needs a signature of its own. + { cp /bin/ps "$work/bin/ps" && codesign -f -s - "$work/bin/ps" 2>/dev/null; } || + refuse "cannot prepare an unprivileged ps" + for i in "${!envargs[@]}"; do + case ${envargs[$i]} in PATH=*) envargs[i]="PATH=$work/bin:${envargs[$i]#PATH=}" ;; esac + done + # Local services that were listening before the command started (a + # development server, an IDE, a published container port or a socket + # proxy) are not the command's: it may connect only to loopback ports it + # opens itself. Every listening port number is held, whatever address + # it listens on. + listening=$(python3 -I "$probe" --listening) || refuse "cannot list the listening TCP ports" + held="" + for port in $listening; do + case $port in *[!0-9]* | "") refuse "unexpected port $port" ;; esac + held="$held (remote ip \"localhost:$port\")" + done + envargs+=("NO_DOCKER_HELD_PORTS=$(printf '%s' "$listening" | tr '\n' ' ')" "NO_DOCKER_OUTER_PID=$$" + "NO_DOCKER_USER_TMP=$tmp") + envargs+=("NO_DOCKER_MARK=$markdir") + # File writes: the run's directory; of /dev only what shells and + # interpreters open for writing (measured, docs/dev-environment.md). + # Nothing in /tmp: bash 3.2 keeps here-documents there when it cannot + # use /var/tmp, whatever $TMPDIR is, so code run inside must not use + # here-documents or here-strings, and no bash in another sandbox can + # have its here-documents changed from inside. Signals, task ports + # (debuggers, vmmap, sample) and process names only within this + # sandbox; no ptrace of any process; no reading other terminals. An + # SBPL error makes sandbox-exec fail, so the command never starts (125). + profile="(version 1) +(allow default) +(deny network-outbound (remote unix-socket)) +(deny network-outbound (remote ip)) +(allow network-outbound (remote ip \"localhost:*\")) +(deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")$held) +(deny file-write*) +(allow file-write* (subpath \"$work\") (literal \"/dev/null\") (literal \"/dev/zero\") (literal \"/dev/tty\") (literal \"/dev/dtracehelper\") (regex #\"^/dev/fd/[0-9]+\$\")) +(deny file-read-data (literal \"$markdir/a\")) +(deny file-read* (regex #\"^/dev/ttys[0-9]+\$\")) +(deny user-preference-write) +(deny file-ioctl (ioctl-command #x80017472)) +(deny process-exec (literal \"/usr/bin/open\") (literal \"/usr/bin/osascript\") (literal \"/bin/launchctl\") (literal \"/usr/bin/ssh\")) +(deny job-creation) +(deny lsopen) +(deny appleevent-send) +(deny mach-lookup (global-name \"com.apple.coreservices.launchservicesd\") (global-name \"com.apple.coreservices.appleevents\") (global-name-regex #\"^com\\.apple\\.lsd\\.\") (global-name \"com.apple.xpc.smd\") (global-name-regex #\"^com\\.apple\\.backgroundtaskmanagement\") (global-name-regex #\"^com\\.apple\\.coreservices\\.sharedfilelistd\") (global-name-regex #\"^com\\.docker\\.\") (global-name-regex #\"^com\\.electron\\.dockerdesktop\\.\")) +(deny system-debug) +(deny mach-task-name mach-task-read mach-task-inspect mach-priv-task-port) +(allow mach-task-name mach-task-read mach-task-inspect mach-priv-task-port (target same-sandbox)) +(deny signal) +(allow signal (target same-sandbox))" + # The supervisor takes over this process: a signal sent to it ends the + # command and everything it left at once, and so does the command's + # end. It removes the run's directory (unless given with --work) and + # the marker directory. + trap - EXIT + shopt -s execfail + exec python3 -I "$probe" --supervise "$markdir" "$work/started" "$remove" -- /usr/bin/sandbox-exec -p "$profile" \ + /usr/bin/env -i "${envargs[@]}" "$BASH" "$here/no-docker.sh" --inner "$@" + rm -rf "$markdir" + [ "$remove" = - ] || rm -rf "$work" + refuse "cannot start the supervisor (python3)" + ;; + Linux) + # L2: the control directory (the launch's state, the namespace record, + # the profile's kernel hash), the fence directory and the profile name. + # Both directories are read-only inside (the seal). + ctldir=$(mktemp -d "$runner_temp/no-docker-ctl.XXXXXX") || refuse "no control directory" + fencedir=$(mktemp -d "$runner_temp/no-docker-fence.XXXXXX") || refuse "no fence directory" + plain "$ctldir" || refuse "unexpected character in $ctldir" + plain "$fencedir" || refuse "unexpected character in $fencedir" + hex=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') + case $hex in *[!0-9a-f]*) refuse "cannot make a random profile name" ;; esac + [ "${#hex}" -eq 32 ] || refuse "cannot make a random profile name" + profile=flowcrm-nd-$hex + load_op=load + if [ -n "$negative" ]; then + # The negative control: an ineffective profile, which the gate must + # refuse. Whatever was asked for, only /usr/bin/true can run. + set -- /usr/bin/true + profile=flowcrm-nd-neg-$hex + load_op=load-negative-control + printf '::warning title=no-docker::negative control: profile %s (the negative template) does not deny host sockets; the command is /usr/bin/true and the status is 125\n' "$profile" + fi + put_state loading || refuse "cannot record the state of the launch" + + # L3: the fence, two sockets of this launch bound outside the namespace + # (stream and datagram), which the command's gate must find refused with + # EACCES. The listeners are this shell's own children; each dies with it + # (PR_SET_PDEATHSIG) and exits unless its parent is this shell. + t=${EPOCHREALTIME//[!0-9]/} + # shellcheck disable=SC2016 # Python, not shell + fence_py=' +import ctypes, os, socket, sys +kind, path, launcher, ready = sys.argv[1:5] +if ctypes.CDLL(None, use_errno=True).prctl(1, 9, 0, 0, 0) != 0 or os.getppid() != int(launcher): + sys.exit(3) +s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM if kind == "stream" else socket.SOCK_DGRAM) +s.bind(path) +if kind == "stream": + s.listen(64) +with open(ready + ".tmp", "w") as f: + f.write("ready\n") +os.rename(ready + ".tmp", ready) +while True: + if kind == "stream": + s.accept()[0].close() + else: + s.recv(65536) +' + pending=stream kept=$! + python3 -I -c "$fence_py" stream "$fencedir/stream" "$$" "$fencedir/stream.ready" "$fencedir/stream.log" 2>&1 & + fspid=$! pending= + proc_stat "$fspid" && fsstart=$ps_start + pending=dgram kept=$! + python3 -I -c "$fence_py" dgram "$fencedir/dgram" "$$" "$fencedir/dgram.ready" "$fencedir/dgram.log" 2>&1 & + fdpid=$! pending= + proc_stat "$fdpid" && fdstart=$ps_start + [ -n "$fsstart" ] && [ -n "$fdstart" ] || refuse "the fence listeners did not start" + wait_file "$fencedir/stream.ready" || refuse "the stream fence listener did not start" + wait_file "$fencedir/dgram.ready" || refuse "the datagram fence listener did not start" + # The positive control: outside, both sockets are reachable. + python3 -I -c ' +import socket, sys +s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +s.settimeout(2) +s.connect(sys.argv[1] + "/stream") +s.close() +s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) +s.sendto(b"no-docker fence", sys.argv[1] + "/dgram") +s.close() +' "$fencedir" &2 || refuse "cannot print the token that stops workflow commands" + # The relays: the command's stdout and stderr are pipes to these two + # children of this shell, which copy to the caller's. + t=${EPOCHREALTIME//[!0-9]/} + pending=relay-o kept=$! + exec {o}> >(exec cat) || refuse "cannot start the stdout relay" + opid=$! pending= + pending=relay-e kept=$! + exec {e}> >(exec cat >&2 {o}>&-) || refuse "cannot start the stderr relay" + epid=$! pending= + relay_ok "$opid" || refuse "the stdout relay is not a cat of this shell" + ostart=$ps_start + relay_ok "$epid" || refuse "the stderr relay is not a cat of this shell" + estart=$ps_start + # No descriptor of this shell but 0 to 2, the relays' write ends and + # the script may reach the namespace (sudo also closes those above 2). + own_fds || refuse "cannot list the descriptors of no-docker.sh" + if [ -n "$fd_dup" ]; then + for x in $fd_dup; do exec {x}>&-; done + own_fds || refuse "cannot list the descriptors of no-docker.sh" + fi + [ -z "$fd_dup$fd_bad" ] || refuse "an unexpected descriptor is open in no-docker.sh:$fd_dup$fd_bad" + fdlist="" + while IFS= read -r line; do + case $line in *" -> "*) x=${line% -> *} && fdlist="$fdlist ${x##* }" ;; esac + done <"$ctldir/fds" + printf 'no-docker: info descriptors of no-docker.sh before the start:%s; relays: stdout pid %s on fd %s, stderr pid %s on fd %s, both children of %s\n' \ + "$fdlist" "$opid" "$o" "$epid" "$e" "$$" >&2 + t_relays=$((${EPOCHREALTIME//[!0-9]/} - t)) + + envargs+=("NO_DOCKER_HOST_NS=${host_ns# }" "NO_DOCKER_HOST_DEVS=${host_devs# }" "NO_DOCKER_PROTECTED=$protected" + "NO_DOCKER_PROFILE=$profile" "NO_DOCKER_CTL=$ctldir" "NO_DOCKER_FENCE=$fencedir") + # pid 1 of the new pid namespace is this root shell. It does not exec: + # it keeps the parent-death signal of --kill-child (a credential change + # would clear it), reaps orphans while it waits, and its exit kills + # everything left in the namespace. Its first action records the + # namespace (CTLDIR/ns), before anything untrusted exists. + # shellcheck disable=SC2016 # expanded by the shell in the namespace + init=' + set -u + PATH=/usr/sbin:/usr/bin:/sbin:/bin + export PATH + die() { printf "no-docker: refused: %s; the command was not run\n" "$1" >&2; exit 125; } + uid=$1 gid=$2 work=$3 probe=$4 cwd=$5 ctl=$6 name=$7 + shift 7 + p=$(readlink /proc/self/ns/pid) && m=$(readlink /proc/self/ns/mnt) || die "cannot read the namespaces" + p=${p#pid:?} m=${m#mnt:?} + p=${p%?} m=${m%?} + case $p in "" | *[!0-9]*) die "unexpected pid namespace link" ;; esac + case $m in "" | *[!0-9]*) die "unexpected mount namespace link" ;; esac + umask 0222 + rm -f "$ctl/.ns.init" && set -C && + printf "pid:%s mnt:%s\n" "$p" "$m" >"$ctl/.ns.init" && set +C && sync "$ctl/.ns.init" && + mv -f "$ctl/.ns.init" "$ctl/ns" || die "cannot record the namespace" + umask 0022 + cd / || die "cannot leave the working directory" + ip link set lo up || die "cannot bring up loopback" + mount -t tmpfs -o mode=0755,nosuid,nodev,noexec no-docker /run || die "cannot hide /run" + mkdir /run/no-docker /run/no-docker/work || die "cannot prepare /run" + mount --bind "$work" /run/no-docker/work || die "cannot keep $work" + if [ -d /var/run ] && [ ! -L /var/run ]; then + mount -t tmpfs -o mode=0755,nosuid,nodev,noexec no-docker /var/run || die "cannot hide /var/run" + fi + for d in /tmp /var/tmp /dev/shm; do + mount -t tmpfs -o mode=1777,nosuid,nodev no-docker "$d" || die "cannot hide $d" + done + mount -t mqueue -o nosuid,nodev,noexec no-docker /dev/mqueue || die "cannot mount a private /dev/mqueue" + mount -t tmpfs -o ro,mode=0555,nosuid,nodev,noexec no-docker /sys/fs/cgroup || die "cannot mask /sys/fs/cgroup" + mkdir -p "$work" && mount --move /run/no-docker/work "$work" || die "cannot move $work into the private temporary directory" + python3 -I "$probe" --seal "$work" || die "cannot make the other mounts read-only" + cd "$cwd" || die "the working directory $cwd is not visible in the isolation" + /usr/bin/python3 -I "$probe" --root-exec "$uid" "$gid" "$name" "$@" & + child=$! + wait "$child" + exit $? + ' + # The state is "entering" before the fork, read back; if it cannot be + # recorded nothing is started. The marker and the last pid are taken in + # one command, so cleanup can tell whether the fork happened ($! is set + # in this shell right after fork(), and traps run between commands). + put_state entering || refuse "cannot record the state of the launch" + t=${EPOCHREALTIME//[!0-9]/} + pending=init kept=$! + sudo -n unshare --net --mount --pid --fork --kill-child --mount-proc --ipc --propagation private -- \ + /bin/sh -c "$init" no-docker-init "$uid" "$gid" "$work" "$probe" "$cwd" "$ctldir" "$profile" \ + -- "${envargs[@]}" -- "$@" &"$o" 2>&"$e" {o}>&- {e}>&- & + spid=$! pending= + proc_stat "$spid" && sstart=$ps_start + put_state "entered spid=$spid" || refuse "cannot record the state of the launch" + exec {o}>&- {e}>&- + o="" e="" + # L9: the command's end (sudo's), then cleanup. + wait "$spid" + rc=$? sudo_reaped=yes + t_run=$((${EPOCHREALTIME//[!0-9]/} - t)) + trap '' INT TERM HUP + normal=yes + exit "$rc" + ;; +esac diff --git a/scripts/ci/no-docker/apparmor.profile b/scripts/ci/no-docker/apparmor.profile new file mode 100644 index 0000000..02c2b12 --- /dev/null +++ b/scripts/ci/no-docker/apparmor.profile @@ -0,0 +1,13 @@ +abi , +profile @NAME@ flags=(enforce) { + file, + unix, + network, + signal, + ptrace (read, readby, trace, tracedby) peer=@NAME@, + deny capability, + deny mount, + deny umount, + deny pivot_root, + deny network vsock, +} diff --git a/scripts/ci/tests/ci-isolation-order-unit.py b/scripts/ci/tests/ci-isolation-order-unit.py new file mode 100644 index 0000000..4613d0a --- /dev/null +++ b/scripts/ci/tests/ci-isolation-order-unit.py @@ -0,0 +1,418 @@ +"""Regressions of the descriptor wrapper rule of ci-isolation-order.py. + +Usage: ci-isolation-order-unit.py PATH_TO_CI_ISOLATION_ORDER + +Prints one "ok"/"FAIL" line per assertion and a SUMMARY line; exit 0 if all +pass. Every command below is workflow text in a temporary fixture: it is only +parsed, never run. +""" + +import importlib.util +import os +import subprocess +import sys +import tempfile + +path = os.path.abspath(sys.argv[1]) +spec = importlib.util.spec_from_file_location("order", path) +order = importlib.util.module_from_spec(spec) +spec.loader.exec_module(order) + +passed = failed = 0 + + +def check(cond, text): + global passed, failed + if cond: + passed += 1 + else: + failed += 1 + print(f"{'ok ' if cond else 'FAIL'} {text}") + + +W = "python3 -I scripts/ci/fdnorm.py --" +ISO = "scripts/iso.test.sh" +root = tempfile.mkdtemp(prefix="order-unit.") + + +def write(rel, text): + full = os.path.join(root, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "w") as f: + f.write(text) + + +STANZA = ('"$here/no-docker.sh" --check >/dev/null 2>&1 ||\n' + ' exec "$here/no-docker.sh" "$0" "$@"\n') +write(ISO, "#!/bin/bash\n" + STANZA) +write("scripts/guard.sh", "#!/bin/bash\nexit 0\n") +write("scripts/ci/fdnorm.py", "# wrapper fixture\n") +write("scripts/ci/no-docker.sh", "#!/bin/bash\n") +write("scripts/ci/tests/no-docker.test.sh", "#!/bin/bash\n") + + +def classify(text): + return order.isolated(root, {"text": text}) + + +# --- recognised: the wrapper around a command that is already isolated +for text in (f"{W} {ISO}", f"{W} scripts/ci/no-docker.sh", f"{W} scripts/ci/tests/no-docker.test.sh"): + iso, mixed, missing = classify(text) + check(iso and not mixed and not missing, f"isolated: {text}") +check(classify(f"{W} {ISO}\n{W} {ISO}")[0], "isolated: two wrapped suites on separate lines") +check(classify(ISO)[0], f"the unwrapped form stays recognised: {ISO}") + +# --- not recognised: everything else that names the wrapper +for text in ( + f"{W} docker", + f"{W} docker ps", + f"{W} docker system df", + f"{W} make", + f"{W} unknown-tool", + f"{W} scripts/guard.sh", + f"{W} scripts/missing.sh", + f"{W} {ISO} extra-argument", + f"{W} bash {ISO}", + f"{W} bash -c 'docker ps'", + f"{W} sh -c {ISO}", + f"{W} env {ISO}", + f"{W} exec {ISO}", + f"{W} python3 {ISO}", + f"{W} ./{ISO}", + f"{W} ../{ISO}", + f"{W} /{ISO}", + f"{W} scripts/../{ISO}", + f"{W} scripts/ci/fdnorm.py", + f"{W} scripts/ci/fdnorm.py -- {ISO}", + f"{W} {ISO}", + f"{W}\t{ISO}", + f"{W} {ISO} &", + f"{W} {ISO} & docker ps", + f"{W} {ISO} >/dev/null", + f"{W} {ISO}>/dev/null", + f"{W} $(docker ps)", + f"{W} `docker ps`", + f"{W} $SUITE", + f"{W} ${{SUITE}}.sh", + f"{W} '{ISO}'", + f'{W} "{ISO}"', + f"{W} scripts/*.sh", + f"python3 scripts/ci/fdnorm.py -- {ISO}", + f"python3 -I -S scripts/ci/fdnorm.py -- {ISO}", + f"python3 -Ic scripts/ci/fdnorm.py -- {ISO}", + f"python -I scripts/ci/fdnorm.py -- {ISO}", + f"/usr/bin/python3 -I scripts/ci/fdnorm.py -- {ISO}", + f"FOO=1 {W} {ISO}", + f"LD_PRELOAD=x {W} {ISO}", + f"env {W} {ISO}", + f"exec {W} {ISO}", + f"sudo {W} {ISO}", + f"python3 -I other/fdnorm.py -- {ISO}", + f"python3 -I ./scripts/ci/fdnorm.py -- {ISO}", + f"python3 -I scripts/ci/fd*.py -- {ISO}", + f"python3 -I scripts/ci/fdnorm.py {ISO}", + f"python3 -I scripts/ci/fdnorm.py - {ISO}", + f"python3 -I scripts/ci/fdnorm.py --x -- {ISO}", + f"scripts/ci/fdnorm.py -- {ISO}", + "python3 -I scripts/ci/fdnorm.py --", + "python3 -I scripts/ci/fdnorm.py", +): + iso, _, _ = classify(text) + check(not iso, f"not isolated: {text!r}") + +# --- a wrapped suite next to any other command is mixed, never isolated +for text in (f"{W} {ISO}\ndocker ps", f"{W} {ISO}; docker ps", f"docker ps && {W} {ISO}", f"docker ps | {W} {ISO}", + f"{W} {ISO}\n{W} docker"): + iso, mixed, _ = classify(text) + check(not iso and mixed, f"mixed: {text!r}") + +# --- a make that cannot be resolved to a recipe is another command, not nothing +write("Makefile", f"iso:\n\t{ISO}\nother:\n\tdocker ps\n") +check(classify("make iso")[0], "isolated: a make target whose recipe is isolated") +for text in (f"{W} {ISO}; make nosuchtarget", f"{W} {ISO}; make -C x y", f"{W} {ISO}; make other", f"{W} {ISO}; FOO=1 make iso"): + iso, mixed, _ = classify(text) + check(not iso and mixed, f"mixed: {text!r}") + +# --- the wrapper needs a path with a directory: execvp searches PATH for a bare name +write("bare.test.sh", "#!/bin/bash\n" + STANZA) +check(not classify(f"{W} bare.test.sh")[0], "not isolated: a bare script name in the wrapper") + +# --- the whole workflow + + +def workflow(*guard_steps): + def job(jid, name, steps): + body = [f" {jid}:", f" name: {name}", " runs-on: ubuntu-24.04", " steps:"] + for i, run in enumerate(steps): + body += [f" - name: Step {i}", f" run: {run}"] + return "\n".join(body) + return ("name: CI\non:\n pull_request:\njobs:\n" + job("guard", "Repository guard", guard_steps) + "\n" + + job("devenv", "Dev environment", [f"{W} {ISO}"]) + "\n") + + +def run_order(*steps): + wf = os.path.join(root, "wf.yml") + with open(wf, "w") as f: + f.write(workflow(*steps)) + return subprocess.run([sys.executable, "-I", path, root, wf], capture_output=True, text=True) + + +def run_text(text): + wf = os.path.join(root, "wf.yml") + with open(wf, "w") as f: + f.write(text) + return subprocess.run([sys.executable, "-I", path, root, wf], capture_output=True, text=True) + + +# Settings that change how an isolated step runs, though its run: line is exact. +GOOD_STEP = f" - name: Suite\n run: {W} {ISO}\n" +JOBS = (" guard:\n name: Repository guard\n{job} runs-on: ubuntu-24.04\n steps:\n" + "{step}" + + " devenv:\n name: Dev environment\n runs-on: ubuntu-24.04\n steps:\n" + GOOD_STEP) +BASE = "name: CI\non:\n pull_request:\n{top}jobs:\n" + JOBS +p = run_text(BASE.format(top="", job="", step=GOOD_STEP)) +check(p.returncode == 0, f"the exact step with no extra keys passes (rc {p.returncode})") +p = run_text(BASE.format(top="", job="", step=GOOD_STEP + " timeout-minutes: 5\n")) +check(p.returncode == 0, "timeout-minutes on an isolated step passes") +for label, kw in ( + ("step shell", {"step": GOOD_STEP + " shell: bash -c 'docker ps; bash {0}'\n"}), + ("step env", {"step": GOOD_STEP + " env:\n LD_PRELOAD: x\n"}), + ("step working-directory", {"step": GOOD_STEP + " working-directory: other\n"}), + ("job env", {"job": " env:\n BASH_ENV: x\n", "step": GOOD_STEP}), + ("job defaults", {"job": " defaults:\n run:\n shell: bash -c 'docker ps; bash {0}'\n", "step": GOOD_STEP}), + ("workflow env", {"top": "env:\n BASH_ENV: x\n", "step": GOOD_STEP}), + ("workflow defaults", {"top": "defaults:\n run:\n shell: sh\n", "step": GOOD_STEP}), +): + p = run_text(BASE.format(**{"top": "", "job": "", **kw})) + check(p.returncode != 0 and "changes how" in p.stdout, f"refused: {label} on an isolated step (rc {p.returncode})") + + +p = run_order("scripts/guard.sh", f"{W} {ISO}") +check(p.returncode == 0, f"a workflow with wrapped suites last passes (rc {p.returncode}): {p.stdout[-200:] if p.returncode else ''}") + +p = run_order("scripts/guard.sh", ISO) +check(p.returncode != 0 and "does not start through" in p.stdout, "an isolated step that is not wrapped fails") + +p = run_order(f"{W} {ISO}", "scripts/guard.sh") +check(p.returncode != 0 and "no step that is not isolated after" in p.stdout, "a step that is not isolated after a wrapped one fails") + +p = run_order(f"{W} {ISO}", f"{W} docker") +check(p.returncode != 0 and "no step that is not isolated after" in p.stdout, "wrapped docker after a wrapped suite fails") + +p = run_order("scripts/guard.sh", f"{W} docker") +check(p.returncode != 0, "a job whose only wrapped command is docker has no isolated step and fails") + +p = run_order("scripts/guard.sh", f"{W} {ISO}; docker ps") +check(p.returncode != 0, "a wrapped suite mixed with docker fails") + +# A failing first suite must fail the step, so a step runs exactly one suite. +for sep in ("||", "|", ";", "&&"): + p = run_order("scripts/guard.sh", f"{W} {ISO} {sep} {W} {ISO}") + check(p.returncode != 0 and "does not start through" in p.stdout, f"two wrapped suites joined by {sep} fail") + +os.rename(os.path.join(root, "scripts/ci/fdnorm.py"), os.path.join(root, "scripts/ci/fdnorm.gone")) +p = run_order("scripts/guard.sh", f"{W} {ISO}") +check(p.returncode != 0 and "wrapper" in p.stdout, "a wrapper that does not exist fails") +os.rename(os.path.join(root, "scripts/ci/fdnorm.gone"), os.path.join(root, "scripts/ci/fdnorm.py")) + +# --- spellings GitHub reads and a line reader may not (FCRM-PARADEV-005) +GATE = " - name: Gate\n run: scripts/guard.sh\n" +SAFE = BASE.format(top="", job="", step=GATE + GOOD_STEP) +p = run_text(SAFE) +check(p.returncode == 0, f"a gate before the suites passes (rc {p.returncode})") + + +def refused(label, text): + p = run_text(text) + check(p.returncode == 1 and "FAIL" in p.stdout and "Traceback" not in p.stderr, f"refused: {label} (rc {p.returncode})") + + +def good(top="", job="", step=None, tail=""): + return BASE.format(top=top, job=job, step=GATE + GOOD_STEP if step is None else step) + tail + + +# Hidden or malformed steps. +refused("a bare dash hides a later step", good(tail=" timeout-minutes: 5\n -\n run: docker ps\n")) +refused("a bare dash with a later key", good(tail=" -\n run: docker ps\n")) +refused("two spaces after the dash", good(tail=" - run: docker ps\n")) +refused("a step at another indent", good(tail=" - run: docker ps\n")) +refused("a list item that is not a mapping", good(tail=" - docker ps\n")) +refused("a nested list in a step", good(tail=" - run: docker ps\n")) +refused("steps in a flow list", SAFE.replace(" steps:\n", " steps: [{run: docker ps}]\n x:\n", 1)) +refused("a job in a flow mapping", good(tail=" hidden: {runs-on: ubuntu-24.04, steps: [{run: docker ps}]}\n")) +refused("a job with steps at the indent of the key", good(tail=" other:\n runs-on: ubuntu-24.04\n steps:\n - run: docker ps\n")) +refused("a decoy step in a multi-line concurrency", good(tail=" other:\n runs-on: ubuntu-24.04\n concurrency: |\n - run: " + f"{W} {ISO}\n steps:\n - run: docker ps\n")) +refused("a tab in the indent", good(tail="\t- run: docker ps\n")) +refused("a job without steps", good(tail=" other:\n runs-on: ubuntu-24.04\n")) +# Run blocks read differently. +for label, run in ( + ("a comment line first in a folded run", f">-\n #\n {W} {ISO}"), + ("a comment line first in a literal run", f"|\n # {W} {ISO}\n {W} {ISO}"), + ("a comment line after a folded run", f">-\n {W} {ISO}\n #"), + ("a keep block indicator", f"|+\n {W} {ISO}"), + ("an indent indicator", f">2\n {W} {ISO}"), + ("an alias", "*suite"), + ("an anchor", f"&suite {W} {ISO}"), + ("a tag", f"!!str {W} {ISO}"), + ("a double-quoted value with an escape", f'"{W} {ISO}\\n"'), + ("a multi-line plain value", f"{W} {ISO}\n docker ps"), + ("a null run", "~"), + ("a mapping run", f"{{a: b}}"), +): + refused(label, good(step=f" - name: Suite\n run: {run}\n")) +refused("a run that is only a comment", good(step=f" - name: Gate\n run: '# nothing'\n - name: Suite\n run: >-\n #\n {W} {ISO}\n")) +# Other ways to write a key. +for key in ("env :", '"env":', "'env':", "env\t:", "? env", "<<: *x", "&a env:", "!!str env:"): + refused(f"workflow key {key!r}", good(top=key + "\n BASH_ENV: x\n")) + refused(f"job key {key!r}", good(job=" " + key + "\n BASH_ENV: x\n")) +for key in ("defaults :", '"defaults":'): + refused(f"workflow key {key!r}", good(top=key + "\n run:\n shell: sh\n")) +refused("step key 'shell :'", good(step=GOOD_STEP + " shell : sh\n")) +refused('step key "run":', good(step=f' - name: Suite\n "run": {W} {ISO}\n')) +refused("a duplicate run key", good(step=GOOD_STEP + " run: docker ps\n")) +refused("a duplicate job key", good(tail=" guard:\n runs-on: ubuntu-24.04\n steps:\n" + GOOD_STEP)) +refused("a duplicate workflow key", good(top="name: other\n")) +refused("a document marker", "---\n" + good()) +refused("a second document", good() + "---\nname: x\n") +refused("a directive", "%YAML 1.1\n" + good()) +for ch, label in (("\x85", "NEL"), ("\u2028", "LINE SEPARATOR"), ("\r", "CR"), ("\x0c", "FF")): + refused(f"a {label} character", good().replace("name: CI", f"name: CI{ch}jobs:", 1)) +# The gates and required jobs cannot be switched off. +for key in ("continue-on-error : true", '"continue-on-error": true', "continue-on-error: true", "if : false", '"if": false', "if: false"): + refused(f"gate step {key!r}", good(step=f" - name: Gate\n {key}\n run: scripts/guard.sh\n" + GOOD_STEP)) + refused(f"required job {key!r}", good(job=f" {key}\n")) +refused("a required job with needs", good(job=" needs: other\n")) +refused("a required job that is a reusable workflow", good(job=" uses: x/y/.github/workflows/z.yml@main\n")) +refused("a required job with a matrix", good(job=" strategy:\n matrix:\n a: [b]\n")) +refused("a required job in a container", good(job=" container: img\n")) +refused("a required job on a computed runner", good().replace("runs-on: ubuntu-24.04", "runs-on: ${{ vars.x }}", 1)) +refused("a required job twice", good(tail=" again:\n name: Dev environment\n runs-on: ubuntu-24.04\n steps:\n" + GOOD_STEP)) +refused("a workflow not run for pull requests", good().replace(" pull_request:\n", " push:\n", 1)) +refused("a path filter", good().replace(" pull_request:\n", " pull_request:\n paths:\n - docs/x\n", 1)) +refused("branches-ignore", good().replace(" pull_request:\n", " pull_request:\n branches-ignore: [main]\n", 1)) +refused("branches without main", good().replace(" pull_request:\n", " pull_request:\n branches: [other]\n", 1)) +refused("another trigger", good().replace(" pull_request:\n", " pull_request:\n workflow_dispatch:\n", 1)) +refused("an unknown workflow key", good(top="x-note: y\n")) +refused("branches as a word that contains main", good().replace(" pull_request:\n", " pull_request:\n branches: domain\n", 1)) +refused("branches as the word main", good().replace(" pull_request:\n", " pull_request:\n branches: main\n", 1)) +for ch, label in (("\xa0", "NO-BREAK SPACE"), ("\u2003", "EM SPACE"), ("\u200b", "ZERO WIDTH SPACE"), ("\u3000", "IDEOGRAPHIC SPACE"), ("\u205f", "MATHEMATICAL SPACE")): + refused(f"a {label} at the end of a quoted run", good(step=f' - name: Suite\n run: "{W} {ISO}{ch}"\n')) + refused(f"a {label} at the end of a plain run", good(step=f" - name: Suite\n run: {W} {ISO}{ch}\n")) +refused("a step with neither run nor uses", good(step=" - name: Nothing\n" + GOOD_STEP)) +refused("a step with both run and uses", good(step=" - name: Both\n uses: a/b@" + "0" * 40 + "\n run: scripts/guard.sh\n" + GOOD_STEP)) +CHECKOUT_STEP = " - name: Checkout\n uses: actions/checkout@" + "0" * 40 + "\n with:\n persist-credentials: false\n" +p = run_text(good(step=CHECKOUT_STEP + GOOD_STEP)) +check(p.returncode == 0, f"the pinned checkout passes (rc {p.returncode})") +refused("an action that is not pinned", good(step=" - name: Mutable\n uses: actions/checkout@main\n with:\n persist-credentials: false\n" + GOOD_STEP)) +refused("another action, though pinned", good(step=" - name: Other\n uses: actions/github-script@" + "0" * 40 + "\n with:\n persist-credentials: false\n" + GOOD_STEP)) +refused("an action that exports BASH_ENV", good(step=" - name: Seed\n uses: actions/github-script@" + "0" * 40 + "\n with:\n script: |\n core.exportVariable('BASH_ENV', '/tmp/x');\n" + GOOD_STEP)) +refused("a checkout that keeps credentials", good(step=CHECKOUT_STEP.replace("false", "true") + GOOD_STEP)) +refused("a checkout of another ref", good(step=CHECKOUT_STEP + " ref: other\n" + GOOD_STEP)) +refused("a checkout without with", good(step=" - name: Checkout\n uses: actions/checkout@" + "0" * 40 + "\n" + GOOD_STEP)) +refused("with on a run step", good(step=" - name: Run\n run: scripts/guard.sh\n with:\n a: b\n" + GOOD_STEP)) +refused("a gate step with env", good(step=" - name: Gate\n env:\n X: y\n run: scripts/guard.sh\n" + GOOD_STEP)) +# Settings written for later steps. +for text in ('echo "BASH_ENV=$PWD/x" >> "$GITHUB_ENV"', 'echo x >> "$GITHUB_PATH"', "echo x >> ${GITHUB_ENV}", 'echo x >> "$GITHUB_"ENV', + "echo x >> $GITHUB_\\ENV", "echo ::add-path::x", "echo ::set-env name=a::b", "tee -a $GITHUB_PATH"): + refused(f"a step that runs {text!r}", good(step=f" - name: Seed\n run: {text}\n" + GOOD_STEP)) +refused("a line continuation in the name", good(step=' - name: Seed\n run: |\n printf x >> "$GITHUB_EN\\\n V"\n' + GOOD_STEP)) +refused("an expression that builds the name", good(step=' - name: Seed\n run: |\n printf x >> "$${{ \'GITHUB\' }}_ENV"\n' + GOOD_STEP)) +refused("an expression in a step", good(step=" - name: Seed\n run: echo ${{ github.ref }}\n" + GOOD_STEP)) +refused("GITHUB_ENV in a folded step", good(step=" - name: Seed\n run: >-\n echo x\n >> $GITHUB_ENV\n" + GOOD_STEP)) +write("scripts/seed.sh", '#!/bin/bash\necho "BASH_ENV=x" >>"$GITHUB_ENV"\n') +refused("a script that writes GITHUB_ENV", good(step=" - name: Seed\n run: scripts/seed.sh\n" + GOOD_STEP)) +write("Makefile", "iso:\n\tscripts/iso.test.sh\nother:\n\tdocker ps\nseed:\n\tscripts/seed.sh\n") +refused("a make target that writes GITHUB_ENV", good(step=" - name: Seed\n run: make seed\n" + GOOD_STEP)) + +# A script is isolated only if nothing runs before its stanza. +write("scripts/late.test.sh", "#!/bin/bash\ndocker ps\n" + STANZA) +check(not classify(f"{W} scripts/late.test.sh")[0], "not isolated: a command before the stanza") +refused("a script with code before its stanza, as an ordinary step", good(step=" - name: Late\n run: scripts/late.test.sh\n" + GOOD_STEP)) +write("scripts/first.test.sh", '#!/bin/bash\n# Comment.\nset -u\n\nhere=$(cd "$(dirname "$0")" && pwd)\n# Comment.\n' + STANZA) +check(classify(f"{W} scripts/first.test.sh")[0], "isolated: only set, here and comments before the stanza") +write("scripts/first-p.test.sh", '#!/bin/bash\nhere=$(cd -P "$(dirname "$0")" && pwd -P)\n' + STANZA) +check(classify(f"{W} scripts/first-p.test.sh")[0], "isolated: the -P form of here") +write("scripts/assign.test.sh", '#!/bin/bash\nhere=$(docker ps)\n' + STANZA) +check(not classify(f"{W} scripts/assign.test.sh")[0], "not isolated: another assignment of here") + +# Harmless spellings of this subset still read. +p = run_text(good(top="# A comment.\n\n", job=" # A comment.\n timeout-minutes: 10\n", + step=" # A comment.\n - name: Checkout\n uses: actions/checkout@" + "0" * 40 + " # v1\n with:\n persist-credentials: false\n" + + " - name: Folded\n run: >-\n scripts/guard.sh\n --flag\n" + + " - name: Literal\n run: |\n scripts/guard.sh\n\n # Not a suite.\n scripts/guard.sh\n" + + " - name: 'Quoted name'\n run: scripts/guard.sh\n" + GOOD_STEP)) +check(p.returncode == 0, f"comments, folded and literal runs, and quoted names read (rc {p.returncode}) {p.stdout[-300:] if p.returncode else ''}") +tree = order.load_workflow("a:\n b: 'c'\n d: \"e\"\n f: [g, h]\n i: j # k\n l: |\n m\n n\n\n # o\n p: >-\n q\n r\n s:\n - t: u\n v: w\n - x: y\n z:\n") +check(tree == {"a": {"b": "c", "d": "e", "f": ["g", "h"], "i": "j", "l": "m\n n\n\n# o", "p": "q r", "s": [{"t": "u", "v": "w"}, {"x": "y"}], "z": None}}, + "the subset reads to the tree YAML gives") + +# --- the six suites of the real workflow +real = tempfile.mkdtemp(prefix="order-real.") +for rel in sum(order.SUITES.values(), []) + ["scripts/ci/no-docker.sh", "scripts/ci/fdnorm.py"]: + full = os.path.join(real, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "w") as f: + f.write("#!/bin/bash\n" + STANZA) +wfdir = os.path.join(real, ".github/workflows") +os.makedirs(wfdir) + + +def real_workflow(guard, devenv): + def steps(paths): + return "".join(f" - name: Step {i}\n run: {W} {path}\n" for i, path in enumerate(paths)) + return ("name: CI\non:\n pull_request:\njobs:\n guard:\n name: Repository guard\n runs-on: ubuntu-24.04\n steps:\n" + steps(guard) + + " devenv:\n name: Dev environment\n runs-on: ubuntu-24.04\n steps:\n" + steps(devenv)) + + +def run_real(guard, devenv): + with open(os.path.join(wfdir, "ci.yml"), "w") as f: + f.write(real_workflow(guard, devenv)) + return subprocess.run([sys.executable, "-I", path, real], capture_output=True, text=True) + + +G, U = order.SUITES["Repository guard"], order.SUITES["Dev environment"] +check(len(G) + len(U) == 6, "six suites are listed") +p = run_real(G, U) +check(p.returncode == 0, f"the six suites in order pass (rc {p.returncode}) {p.stdout[-200:] if p.returncode else ''}") +for label, guard, devenv in ( + ("a suite left out", G[:-1], U), + ("the last suite left out", G, []), + ("suites in another order", G[::-1], U), + ("a suite twice", G + G[-1:], U), + ("a suite in the other job", G[1:], U + G[:1]), +): + p = run_real(guard, devenv) + check(p.returncode == 1, f"refused: {label} (rc {p.returncode})") + +# --- the real workflow reads the same here and in PyYAML, where that exists +repo = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(path)))) +real_ci = os.path.join(repo, ".github/workflows/ci.yml") +try: + import yaml +except ImportError: + yaml = None +if yaml is None or not os.path.isfile(real_ci): + print("skip the comparison with PyYAML: PyYAML or the real workflow is not available") +else: + def text_tree(node): + if isinstance(node, dict): + return {("on" if k is True else str(k)): text_tree(v) for k, v in node.items()} + if isinstance(node, list): + return [text_tree(v) for v in node] + if node is None: + return None + return str(node).lower() if isinstance(node, bool) else str(node) + with open(real_ci) as f: + text = f.read() + mine = order.load_workflow(text) + theirs = text_tree(yaml.safe_load(text)) + # YAML keeps the trailing line break of a literal block; the shell does not care. + def trimmed(node): + if isinstance(node, dict): + return {k: trimmed(v) for k, v in node.items()} + if isinstance(node, list): + return [trimmed(v) for v in node] + return node.rstrip("\n") if isinstance(node, str) else node + check(trimmed(mine) == trimmed(theirs), "the real ci.yml reads the same as in PyYAML") + +print(f"SUMMARY {passed} ok, {failed} failed") +sys.exit(0 if failed == 0 else 1) diff --git a/scripts/ci/tests/ci-isolation-order.py b/scripts/ci/tests/ci-isolation-order.py new file mode 100644 index 0000000..75f1258 --- /dev/null +++ b/scripts/ci/tests/ci-isolation-order.py @@ -0,0 +1,536 @@ +"""Check that isolated steps come last in every CI job (static). + +Usage: ci-isolation-order.py REPO_ROOT [WORKFLOW] (default .github/workflows/ci.yml) + +A step is isolated when every command it runs (in its `run:` text, or in +the recipe of a `make` target it runs) is scripts/ci/no-docker.sh, its +test harness scripts/ci/tests/no-docker.test.sh, or a script with the +recognised re-exec stanza: it runs code under test, which may leave files +or processes for later steps. A step that mixes such commands with others +fails. +The descriptor wrapper scripts/ci/fdnorm.py is transparent to this rule only +in the exact form "python3 -I scripts/ci/fdnorm.py -- PATH", with nothing +else on the command: the classification is then that of PATH. Any other +command that names the wrapper is not isolated, and the wrapper never makes +a command isolated by itself. +Code under test cannot write anything on the Linux runner's file systems +except private temporary directories, but the order is the second line: +after the first isolated step of a job, only isolated steps may follow. + +Also checked: + - the workflow is in the strict subset of YAML that load_workflow reads. A + spelling it does not know is an error, not something to skip: GitHub would + read `env :`, a bare `-` or a comment line in a block scalar differently + from a reader that matches line prefixes; + - the workflow, its jobs and its steps have only the keys listed below, the + workflow runs for pull requests without path filters, and no job has `needs:` + or another key that could skip a required check; + - the required jobs "Repository guard" and "Dev environment" exist once and + each has an isolated step; + - in .github/workflows/ci.yml, they run exactly the suites in SUITES, in order; + - every isolated step of those jobs is exactly one wrapped command, and the + wrapper exists; + - no isolated step has a key other than name, run and timeout-minutes, and + no env or defaults apply to it from its job or the workflow; + - no step writes to $GITHUB_ENV or $GITHUB_PATH, in its text or in a script it + runs: that would reach later isolated steps as settings. The only action + allowed is actions/checkout, pinned to a commit, with persist-credentials: false; + - no run text has a ${{ }} expression, which GitHub expands before the shell; + - a script counts as isolated only if its re-exec stanza comes before anything + but comments, set options and the assignment of $here; + - no job or step has an `if:` condition (one could run a non-isolated + step after a failed isolated one, or skip a required check) or a + `continue-on-error:` key (a failed step, a security gate included, + would no longer fail its job); + - every tracked file that mentions no-docker.sh is either part of the + isolation itself, documentation, the workflow, or has the re-exec + stanza in its recognised form, so a suite isolated another way is + noticed instead of being classified as not isolated. +This reads the workflow's text. A script can still compute such a name, and the +runner has not been consulted: actionlint and GitHub read the same file separately. +Exit 0 if all hold; prints one line per check. +""" + +import os +import re +import subprocess +import sys + +STANZA = re.compile(r'no-docker\.sh" --check >/dev/null 2>&1 \|\|\n\s+exec "[^"\n]*/no-docker\.sh" ') +OWN = {"scripts/ci/no-docker.sh", "scripts/ci/no-docker-probe.py", "scripts/ci/tests/no-docker.test.sh", + "scripts/ci/tests/ci-isolation-order.py", ".github/workflows/ci.yml", + "scripts/ci/tests/no-docker-static.py", "scripts/ci/no-docker-aa.py", + "scripts/ci/tests/ci-isolation-order-unit.py"} +WRAPPER = "scripts/ci/fdnorm.py" +# The only recognised use of the wrapper. No component starts with a dot, so +# neither ./ nor ../ can reach a script outside the checkout's own paths. A +# directory is required: execvp searches PATH for a bare name, which need not be +# the file classified here. +WRAPPED = re.compile(r"python3 -I scripts/ci/fdnorm\.py -- ((?:[\w-][\w.-]*/)+[\w-][\w.-]*\.(?:sh|py))") +SEPARATORS = r"\n|;|&&|\|\||\|" +# Keys an isolated step may have. Others, such as shell, env or +# working-directory, change how its exact run: line behaves. +STEP_KEYS = {"name", "run", "timeout-minutes"} +# Workflow and job keys that do the same for every step below them. +INHERITED = ("env", "defaults") +SCRIPT = re.compile(r"(?", ">-") +# A line break or control character that a line-by-line reader and YAML see differently. +# Unicode spaces too: Python's strip removes them, and YAML and the shell keep them. +BAD_CHAR = re.compile("[\\x00-\\x09\\x0b-\\x1f\\x7f-\\xa0\\u1680\\u180e\\u2000-\\u200f\\u2028-\\u202f\\u205f-\\u2064\\u3000\\ufeff]") + + +def line_indent(line): + return len(line) - len(line.lstrip(" ")) + + +def is_blank(line): + stripped = line.strip() + return not stripped or stripped.startswith("#") + + +def scalar(value, no): + """A one-line value. Always a string, or a list for a flow list of plain words.""" + value = value.strip(" ") + quoted = re.fullmatch(r"(?:'([^']*)'|\"([^\"\\\\]*)\")(?: +#.*)?", value) + if quoted: + return quoted.group(1) if quoted.group(1) is not None else quoted.group(2) + value = re.sub(r"(^| +)#.*$", "", value) + if not value: + raise Unreadable(f"line {no}: a value that is only a comment") + if FLOW_LIST.fullmatch(value): + return [w.strip() for w in value[1:-1].split(",")] + if value[0] in "&*!|>'\"%@`{[]},?#~" or value in ("-", "?", ":") or value.startswith(("- ", "? ", ": ")): + raise Unreadable(f"line {no}: value {value!r} needs more of YAML than this file reads") + if ": " in value or value.endswith(":"): + raise Unreadable(f"line {no}: value {value!r} has a colon that YAML may read as a key") + return value + + +class Reader: + def __init__(self, text): + bad = BAD_CHAR.search(text) + if bad: + raise Unreadable(f"character {ord(bad.group(0)):#x} is not allowed") + self.lines = text.split("\n") + self.i = 0 + + def skip(self): + while self.i < len(self.lines) and is_blank(self.lines[self.i]): + self.i += 1 + + def more(self, indent): + """Whether the next line that holds content is at this indent; deeper is an error.""" + self.skip() + if self.i >= len(self.lines): + return False + n = line_indent(self.lines[self.i]) + if n > indent: + raise Unreadable(f"line {self.i + 1}: unexpected indent") + return n == indent + + def block(self, indent): + self.skip() + if self.lines[self.i].lstrip(" ").startswith("-"): + return self.sequence(indent) + return self.mapping(indent) + + def sequence(self, indent): + items = [] + while self.more(indent): + line = self.lines[self.i] + rest = line[indent:] + if not rest.startswith("- ") or rest[2:3] == " " or not KEY.fullmatch(rest[2:]): + raise Unreadable(f"line {self.i + 1}: a list item must be written as \"- key: value\"") + self.lines[self.i] = " " * (indent + 2) + rest[2:] + items.append(self.mapping(indent + 2)) + return items + + def mapping(self, indent): + result = {} + while self.more(indent): + no = self.i + 1 + m = KEY.fullmatch(self.lines[self.i][indent:]) + if not m: + raise Unreadable(f"line {no}: not a plain \"key: value\": {self.lines[self.i].strip()!r}") + key, value = m.group(1), m.group(2) + if key in result: + raise Unreadable(f"line {no}: duplicate key {key}") + self.i += 1 + value = (value or "").strip(" ") + if value in BLOCK_HEADERS: + result[key] = self.block_scalar(indent, value, no) + elif not value or value.startswith("#"): + self.skip() + deeper = self.i < len(self.lines) and line_indent(self.lines[self.i]) > indent + result[key] = self.block(line_indent(self.lines[self.i])) if deeper else None + else: + result[key] = scalar(value, no) + return result + + def block_scalar(self, indent, header, no): + body = [] + width = None + while self.i < len(self.lines): + line = self.lines[self.i] + if not line.strip(): + body.append("") + self.i += 1 + continue + n = line_indent(line) + if n <= indent: + break + if width is None: + width = n + if n < width: + raise Unreadable(f"line {self.i + 1}: less indented than its block scalar") + body.append(line[width:]) + self.i += 1 + while body and not body[-1]: + body.pop() + if not body: + raise Unreadable(f"line {no}: empty block scalar") + if header[0] == ">": + if any(not b or b.startswith(" ") for b in body): + raise Unreadable(f"line {no}: a folded scalar with a blank or deeper line") + return " ".join(body) + return "\n".join(body) + + +def load_workflow(text): + """The workflow as dicts, lists and strings, or Unreadable. + + This is a strict subset of YAML, not a YAML parser: the workflow is checked with + `python3 -I` on a runner where PyYAML is not known to exist and nothing may be + installed, and refusing every other spelling cannot read one differently from + GitHub. The regressions compare it with PyYAML where that is available.""" + reader = Reader(text) + tree = reader.mapping(0) + reader.skip() + if reader.i < len(reader.lines): + raise Unreadable(f"line {reader.i + 1}: not a plain \"key: value\" at the top level") + return tree + + +def parse(path): + """{job id: {"name": str, "steps": [{"name": str, "text": str, ...}], ...}} of a + workflow. Keys the rules care about are kept for main to report.""" + with open(path, encoding="utf-8") as f: + return jobs_of(load_workflow(f.read())) + + +def jobs_of(tree): + jobs = {} + inherited = [f"workflow {k}" for k in INHERITED if k in tree] + raw_jobs = tree.get("jobs") + if not isinstance(raw_jobs, dict) or not raw_jobs: + raise Unreadable("the workflow has no jobs mapping") + for jid, raw in raw_jobs.items(): + if not isinstance(raw, dict) or not isinstance(raw.get("steps"), list): + raise Unreadable(f"job {jid} has no list of steps") + job = {"name": raw.get("name") if isinstance(raw.get("name"), str) else "", "keys": set(raw), "steps": [], + "if": "if" in raw, "coe": "continue-on-error" in raw, "raw": raw, + "inherited": inherited + [f"job {k}" for k in INHERITED if k in raw]} + for rs in raw["steps"]: + run = rs.get("run") + job["steps"].append({"name": rs.get("name") if isinstance(rs.get("name"), str) else "", + "text": run if isinstance(run, str) else "", "keys": set(rs), "raw": rs, + "if": "if" in rs, "coe": "continue-on-error" in rs}) + jobs[jid] = job + return jobs + + +def make_recipe(root, target): + try: + lines = open(os.path.join(root, "Makefile")).read().splitlines() + except OSError: + return "" + out, inside = [], False + for line in lines: + if re.match(rf"^{re.escape(target)}\s*:", line): + inside = True + continue + if inside: + if line.startswith("\t"): + out.append(line) + else: + break + return "\n".join(out) + + +def commands(text): + """The program each simple command of a shell text runs: the first word + after variable assignments and an interpreter (bash, sh, python3, exec, + env) of every command separated by a newline, ;, &&, || or |. A command + in the exact form of WRAPPED counts as PATH; any other command that names + the wrapper counts as the wrapper, which has no stanza and is not isolated.""" + words = [] + for part in re.split(SEPARATORS, text): + wrapped = WRAPPED.fullmatch(part.strip()) + if wrapped: + words.append(wrapped.group(1)) + continue + if "fdnorm" in part or "fd*" in part: + words.append(WRAPPER) + continue + toks = part.split() + while toks and (re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", toks[0]) or toks[0] in ("exec", "env", "then", "do", "!") + or toks[0].lstrip("@") in ("bash", "sh", "python3", "/bin/bash", "/bin/sh")): + toks = toks[1:] + while toks and toks[0].startswith("-") and toks[0] != "-": + toks = toks[1:] # interpreter options such as -I or -e + if toks: + words.append(toks[0]) + return words + + +def step_commands(root, text): + """Every program a step runs: its own commands and those of the + recipes of the make targets it runs. A make command that is not plainly + "make TARGET..." with a recipe for each target stays the command make, + which is not isolated.""" + words = [] + for part in re.split(SEPARATORS, text): + found = commands(part) + plain = re.fullmatch(r"make((?:\s+[A-Za-z][\w-]*)+)", part.strip()) + recipes = [make_recipe(root, t) for t in plain.group(1).split()] if plain and found == ["make"] else [] + if recipes and all(recipes): + for recipe in recipes: + words += commands(recipe.replace("\t", " ")) + else: + words += found + return [w[2:] if w.startswith("./") else w for w in words] + + +def stanza_first(content): + """The re-exec stanza is there, and nothing ran before it.""" + m = STANZA.search(content) + if not m: + return False + start = content.rfind("\n", 0, m.start()) + 1 + return all(BEFORE_STANZA.fullmatch(line.strip()) for line in content[:start].split("\n")) + + +def propagates(root, step): + """Whether the step writes settings for later steps: in its text, or in a script + it runs. Quotes and backslashes do not hide the name from this.""" + texts = [step["text"]] + for w in step_commands(root, step["text"]): + if SCRIPT.fullmatch(w): + try: + with open(os.path.join(root, w), encoding="utf-8", errors="replace") as f: + texts.append(f.read()) + except OSError: + pass + return any(PROPAGATES.search(re.sub(r"[\"'\\\\\n]", "", t)) for t in texts) + + +def late_stanzas(root, step): + """Scripts a step runs that have the stanza after other code: such a script is + neither isolated nor ordinary, so it is refused rather than reclassified.""" + found = [] + for w in step_commands(root, step["text"]): + if SCRIPT.fullmatch(w): + try: + with open(os.path.join(root, w), encoding="utf-8", errors="replace") as f: + content = f.read() + except OSError: + continue + if STANZA.search(content) and not stanza_first(content): + found.append(w) + return found + + +def check_trigger(tree): + """The workflow runs for pull requests, and no filter can skip it for some of them.""" + on = tree.get("on") + problems = [] + if not isinstance(on, dict) or "pull_request" not in on: + return ["it does not run for pull requests"] + for event, spec in on.items(): + if event not in ("pull_request", "push"): + problems.append(f"event {event}") + elif spec is not None and (not isinstance(spec, dict) or set(spec) != {"branches"} or not isinstance(spec["branches"], list) + or "main" not in spec["branches"]): + problems.append(f"{event} has filters other than branches including main") + return problems + + +def check_structure(tree, jobs): + extra = sorted(set(tree) - WORKFLOW_KEYS - set(INHERITED)) + report(not extra, "the workflow has only known keys" + (f" (also: {', '.join(extra)})" if extra else "")) + problems = check_trigger(tree) + report(not problems, "the workflow runs for every pull request" + (f" ({'; '.join(problems)})" if problems else "")) + for jid, job in jobs.items(): + label = job["name"] or jid + extra = sorted(job["keys"] - JOB_KEYS - set(INHERITED) - {"if", "continue-on-error"}) + report(not extra, f"{label}: the job has only known keys" + (f" (also: {', '.join(extra)}; needs and others can skip it)" if extra else "")) + runner = job["raw"].get("runs-on") + report(isinstance(runner, str) and bool(RUNNER.fullmatch(runner)), f"{label}: runs on a fixed Ubuntu image") + for step in job["steps"]: + name = f"{label} / {step['name']}" + raw = step["raw"] + extra = sorted(step["keys"] - ANY_STEP_KEYS - {"if", "continue-on-error"}) + report(not extra, f"{name}: the step has only known keys" + (f" (also: {', '.join(extra)})" if extra else "")) + # GitHub expands an expression before the shell sees the text, so nothing + # read here is what would run. + report("${{" not in step["text"], f"{name}: has no expression in its run text") + kinds = ("run" in raw) + ("uses" in raw) + report(kinds == 1 and isinstance(raw.get("run", ""), str), f"{name}: has one run text or one action") + if "uses" in raw: + report(isinstance(raw["uses"], str) and bool(CHECKOUT.fullmatch(raw["uses"])), f"{name}: the action is actions/checkout, pinned to a commit") + report(raw.get("with") == {"persist-credentials": "false"}, f"{name}: the action has exactly persist-credentials: false") + elif "with" in raw: + report(False, f"{name}: has with only for an action") + + +def isolated(root, step): + """(isolated, mixed, missing scripts) of a step. A command is isolated + when it is scripts/ci/no-docker.sh, the isolation's own test harness, or + a script with the recognised re-exec stanza; any other command (docker, + make help, a trusted guard) is not. A step is isolated when all of its + commands are; one that mixes both is reported.""" + iso, other, missing = [], [], [] + for w in step_commands(root, step["text"]): + if not SCRIPT.fullmatch(w): + other.append(w) + continue + if w in ("scripts/ci/no-docker.sh", HARNESS): + iso.append(w) + continue + try: + content = open(os.path.join(root, w), encoding="utf-8", errors="replace").read() + except OSError: + missing.append(w) + continue + (iso if stanza_first(content) else other).append(w) + return bool(iso) and not other, bool(iso) and bool(other), missing + + +def starts_through_wrapper(text): + """The whole step is one command in the exact form of WRAPPED. One suite per + step: a second one joined by || or | could hide the first one's failure.""" + return bool(WRAPPED.fullmatch(text)) + + +def tracked(root): + try: + out = subprocess.run(["git", "-C", root, "ls-files", "-z"], capture_output=True, check=True).stdout + return [p for p in out.decode().split("\0") if p] + except (OSError, subprocess.CalledProcessError): + found = [] + for d, dirs, files in os.walk(root): + dirs[:] = [x for x in dirs if x != ".git"] + found += [os.path.relpath(os.path.join(d, f), root) for f in files] + return found + + +def main(): + root = os.path.abspath(sys.argv[1]) + real = len(sys.argv) <= 2 + workflow = os.path.join(root, ".github/workflows/ci.yml") if real else sys.argv[2] + try: + with open(workflow, encoding="utf-8") as f: + tree = load_workflow(f.read()) + jobs = jobs_of(tree) + except (Unreadable, OSError, UnicodeDecodeError) as e: + report(False, f"the workflow can be read strictly: {e}") + return 1 + report(True, "the workflow can be read strictly") + check_structure(tree, jobs) + names = [j["name"] for j in jobs.values()] + for required in REQUIRED: + report(names.count(required) == 1, f"required job {required!r} exists once") + report(not any(j["if"] or any(s["if"] for s in j["steps"]) for j in jobs.values()), + "no job or step has an if: condition") + coe = [s["name"] or job["name"] for job in jobs.values() for s in [{"name": ""}] * job["coe"] + job["steps"] + if s.get("coe", job["coe"] and not s["name"])] + report(not coe, "no job or step has continue-on-error" + (f" ({', '.join(repr(c) for c in coe)})" if coe else "")) + report(os.path.isfile(os.path.join(root, WRAPPER)), f"the descriptor wrapper {WRAPPER} exists") + for jid, job in jobs.items(): + first = None + late = [] + for i, step in enumerate(job["steps"]): + iso, mixed, missing = isolated(root, step) + report(not missing, f"{job['name'] or jid} / {step['name']}: the scripts it runs exist" + + (f" (missing {' '.join(missing)})" if missing else "")) + report(not mixed, f"{job['name'] or jid} / {step['name']}: does not mix isolated and other commands") + report(not propagates(root, step), f"{job['name'] or jid} / {step['name']}: writes nothing to GITHUB_ENV or GITHUB_PATH") + late_scripts = late_stanzas(root, step) + report(not late_scripts, f"{job['name'] or jid} / {step['name']}: no script runs code before its stanza" + + (f" ({' '.join(late_scripts)})" if late_scripts else "")) + if iso and job["name"] in REQUIRED: + through = starts_through_wrapper(step["text"]) + report(through, f"{job['name'] or jid} / {step['name']}: " + + ("starts through the descriptor wrapper" if through else "does not start through the descriptor wrapper")) + if iso: + changes = sorted(step["keys"] - STEP_KEYS) + job["inherited"] + report(not changes, f"{job['name'] or jid} / {step['name']}: nothing changes how the step runs" + + (f" (changes how it runs: {', '.join(changes)})" if changes else "")) + if iso and first is None: + first = step["name"] + elif not iso and first is not None: + late.append(step["name"]) + report(not late, f"{job['name'] or jid}: no step that is not isolated after the first isolated one" + + (f" ({first!r}, then {', '.join(repr(s) for s in late)})" if late else "")) + if job["name"] in REQUIRED: + report(first is not None, f"{job['name']}: has isolated steps (first: {first})") + if real and job["name"] in SUITES: + ran = [m.group(1) for m in (WRAPPED.fullmatch(s["text"]) for s in job["steps"]) if m] + report(ran == SUITES[job["name"]], f"{job['name']}: runs exactly its wrapped suites, in order" + + (f" (found {' '.join(ran)})" if ran != SUITES[job["name"]] else "")) + unknown = [] + for p in tracked(root): + if p in OWN or p.endswith(".md"): + continue + try: + content = open(os.path.join(root, p), encoding="utf-8", errors="replace").read() + except OSError: + continue + if "no-docker.sh" in content and not STANZA.search(content): + unknown.append(p) + report(not unknown, "every other file that mentions no-docker.sh has the recognised re-exec stanza" + + (f" (not recognised: {' '.join(unknown)})" if unknown else "")) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/tests/fdnorm-unit.py b/scripts/ci/tests/fdnorm-unit.py new file mode 100644 index 0000000..1515739 --- /dev/null +++ b/scripts/ci/tests/fdnorm-unit.py @@ -0,0 +1,208 @@ +"""Unit tests of scripts/ci/fdnorm.py. + +Usage: fdnorm-unit.py PATH_TO_FDNORM + +Prints one "ok"/"FAIL" line per assertion and a SUMMARY line; exit 0 if all +pass. The commands it starts are this Python printing its own descriptors. +""" + +import errno +import importlib.util +import os +import stat +import subprocess +import sys + +path = os.path.abspath(sys.argv[1]) +spec = importlib.util.spec_from_file_location("fdnorm", path) +fdnorm = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fdnorm) + +passed = failed = 0 +PY = sys.executable +FDDIR = fdnorm.FDDIR +# Prints the descriptors COMMAND has, without the listing's own. +LIST = ("import os\n" + f"names = os.listdir({FDDIR!r})\n" + "def live(n):\n" + " try:\n" + " os.fstat(int(n)); return True\n" + " except OSError:\n" + " return False\n" + "print(' '.join(sorted((n for n in names if live(n)), key=int)))\n") + + +def check(cond, text): + global passed, failed + if cond: + passed += 1 + else: + failed += 1 + print(f"{'ok ' if cond else 'FAIL'} {text}") + + +def run(args, **kw): + return subprocess.run([PY, "-I", path] + args, capture_output=True, text=True, **kw) + + +# --- real runs +r, w = os.pipe() +high = os.dup2(r, 142, inheritable=True) +p = run(["--", PY, "-I", "-c", LIST], pass_fds=(142,)) +check(p.returncode == 0, f"a command runs with an inherited pipe above 2 (rc {p.returncode})") +check(p.stdout.split() == ["0", "1", "2"], f"the command has only 0 1 2 (got {p.stdout.strip()!r})") +check("closed 142" in p.stderr, "the closed descriptor is reported") +check(p.stderr.startswith("ci-fd: before: "), "the report has its prefix") +check("/" not in p.stderr, "the report names no path") +os.close(high) +os.close(r) +os.close(w) + +p = run(["--", PY, "-I", "-c", LIST]) +check(p.returncode == 0 and p.stdout.split() == ["0", "1", "2"], "with nothing to close the command still runs") + +p = run(["--", PY, "-I", "-c", "import sys; sys.exit(7)"]) +check(p.returncode == 7, f"the command's own status is returned (rc {p.returncode})") + +child = subprocess.Popen([PY, "-I", path, "--", PY, "-I", "-c", "import os; print(os.getpid())"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) +out = child.communicate()[0].strip() +check(out == str(child.pid), f"the command replaces the wrapper, same pid ({out} and {child.pid})") + +for args, label in (([], "no arguments"), (["--"], "no command"), ([PY], "no --"), ([PY, "--", "x"], "-- not first")): + p = run(args) + check(p.returncode == 2, f"usage refused, {label} (rc {p.returncode})") + check("the command was not run" in p.stderr, f"the refusal says the command did not run, {label}") + +p = run(["--", "/nonexistent/fdnorm-test-command"]) +check(p.returncode == 127, f"a command that cannot be executed is 127 (rc {p.returncode})") + +p = subprocess.run([PY, "-I", path, "--", PY, "-I", "-c", "print('ran')"], capture_output=True, text=True, + stdin=subprocess.DEVNULL, preexec_fn=lambda: os.close(0)) +check(p.returncode == 2 and "ran" not in p.stdout, f"a closed fd 0 is refused (rc {p.returncode})") + + +# --- the failure paths, through a replaced System +def identity(fd): + return os.stat_result((stat.S_IFIFO | 0o600, 100 + fd, 1, 1, 0, 0, 0, 0, 0, 0)) + + +class Fake(fdnorm.System): + """Three standard descriptors plus OPEN, until closerange runs.""" + + def __init__(self): + self.open = {0, 1, 2, 142} + self.closed = False + self.ran = None + self.out = b"" + self.limit = 1024 + self.keep_after_close = set() + self.names = None + self.change = None + self.flagset = {} + self.signals = [] + self.fail_write = False + self.listerr = None + self.fstaterr = {} + self.fcntlerr = {} + + def listdir(self, _): + if self.listerr: + raise OSError(self.listerr, "x") + return list(self.names) if self.names is not None else [str(fd) for fd in sorted(self.open)] + + def fstat(self, fd): + if fd in self.fstaterr: + raise OSError(self.fstaterr[fd], "x") + if fd not in self.open: + raise OSError(errno.EBADF, "x") + return identity(fd + (1000 if self.closed and self.change == fd else 0)) + + def flags(self, fd): + if fd in self.fcntlerr: + raise OSError(self.fcntlerr[fd], "x") + return self.flagset.get(fd, (os.O_RDWR, 0)) + + def closerange(self, low, high): + self.closed = True + self.open = {fd for fd in self.open if fd < low or fd >= high or fd in self.keep_after_close} + + def write(self, fd, data): + if self.fail_write: + raise OSError(errno.EPIPE, "x") + self.out += bytes(data) + return len(data) + + def execvp(self, name, argv): + self.ran = argv + + def signal(self, num, handler): + self.signals.append(num) + + def hard_limit(self): + return self.limit + + def process(self): + return "pid 1 ppid 1 uid 1 euid 1 start -" + + +ARGV = ["fdnorm.py", "--", "suite"] + + +f = Fake() +rc = fdnorm.main(ARGV, f) # the fake exec returns, so main ends as after a failed exec +check(rc == 127, f"after the exec call main returns 127 (status {rc})") +check(f.ran == ["suite"], "COMMAND is what is executed") +check(b"closed 142" in f.out, "142 is reported closed") + +for label, setup in ( + ("a listing that fails", lambda f: setattr(f, "listerr", errno.EACCES)), + ("a name that is not a number", lambda f: setattr(f, "names", ["0", "1", "2", "x"])), + ("a descriptor kept open after closerange", lambda f: f.keep_after_close.add(142)), + ("an fstat error", lambda f: f.fstaterr.__setitem__(142, errno.EIO)), + ("an EBADF on fd 1", lambda f: f.fstaterr.__setitem__(1, errno.EBADF)), + ("an fcntl error", lambda f: f.fcntlerr.__setitem__(142, errno.EIO)), + ("fd 2 missing", lambda f: f.open.discard(2)), + ("fd 1 close-on-exec", lambda f: f.flagset.__setitem__(1, (os.O_RDWR, 1))), + ("an infinite limit", lambda f: setattr(f, "limit", fdnorm.resource.RLIM_INFINITY)), + ("a limit below 3", lambda f: setattr(f, "limit", 2)), + ("a limit above the maximum", lambda f: setattr(f, "limit", fdnorm.MAX_LIMIT + 1)), + ("a limit that is not an integer", lambda f: setattr(f, "limit", 1.5)), + ("a descriptor at the limit", lambda f: setattr(f, "limit", 142)), + ("a standard descriptor that changes", lambda f: setattr(f, "change", 1)), + ("too many descriptors", lambda f: setattr(f, "names", [str(n) for n in range(fdnorm.MAX_FDS + 2)])), +): + f = Fake() + setup(f) + rc = fdnorm.main(ARGV, f) + check(rc == 2 and f.ran is None, f"refused: {label} (status {rc}, ran {f.ran})") + +f = Fake() +f.fail_write = True +rc = fdnorm.main(ARGV, f) +check(rc == 2 and f.ran is None, "a report that cannot be written stops the run") + +f = Fake() +f.names = ["0", "1", "2", "9"] +f.fstaterr[9] = errno.EBADF +rc = fdnorm.main(ARGV, f) +check(f.ran == ["suite"], "the listing's own closed descriptor is allowed") + +f = Fake() +f.names = ["0", "1", "2", "8", "9"] +f.fstaterr[8] = f.fstaterr[9] = errno.EBADF +rc = fdnorm.main(ARGV, f) +check(rc == 2 and f.ran is None, "two closed descriptors in a listing are refused") + +f = Fake() +fdnorm.main(ARGV, f) +check(fdnorm.signal.SIGPIPE in f.signals and fdnorm.signal.SIGXFSZ in f.signals, "SIGPIPE and SIGXFSZ are reset") + +src = open(path, encoding="utf-8").read() +check("no-docker" not in src, "the wrapper does not name the launcher's script") +code = src[src.index("import errno"):] +check("execvp" in code and "import subprocess" not in code and "os.fork" not in code and "os.system" not in code, + "the wrapper executes COMMAND and starts no child") + +print(f"SUMMARY {passed} ok, {failed} failed") +sys.exit(0 if failed == 0 else 1) diff --git a/scripts/ci/tests/guard-dockerfile-frontend.test.sh b/scripts/ci/tests/guard-dockerfile-frontend.test.sh index 623db6a..0deb7d2 100755 --- a/scripts/ci/tests/guard-dockerfile-frontend.test.sh +++ b/scripts/ci/tests/guard-dockerfile-frontend.test.sh @@ -10,6 +10,10 @@ set -u here=$(cd "$(dirname "$0")" && pwd) +# The suite runs the guard, or a modified or mutated copy of it (GUARD=), +# so it runs only where no container engine can be reached. +"$here/../no-docker.sh" --check >/dev/null 2>&1 || + exec "$here/../no-docker.sh" "$BASH" "$here/guard-dockerfile-frontend.test.sh" "$@" guard=${GUARD:-"$here/../guard-dockerfile-frontend.sh"} real=$(git -C "$here" rev-parse --show-toplevel) || exit 2 work=$(mktemp -d "${TMPDIR:-/tmp}/guard-frontend-tests.XXXXXX") || exit 2 diff --git a/scripts/ci/tests/guard-edge-config.test.sh b/scripts/ci/tests/guard-edge-config.test.sh index 5963a6f..1cfe4b1 100755 --- a/scripts/ci/tests/guard-edge-config.test.sh +++ b/scripts/ci/tests/guard-edge-config.test.sh @@ -14,6 +14,10 @@ set -u here=$(cd "$(dirname "$0")" && pwd) +# The suite runs the guard, or a modified or mutated copy of it (GUARD=), +# so it runs only where no container engine can be reached. +"$here/../no-docker.sh" --check >/dev/null 2>&1 || + exec "$here/../no-docker.sh" "$BASH" "$here/guard-edge-config.test.sh" "$@" guard=${GUARD:-"$here/../guard-edge-config.sh"} adapted=${ADAPTED:-"$here/../edge-adapted-config.jq"} real=$(git -C "$here" rev-parse --show-toplevel) || exit 2 diff --git a/scripts/ci/tests/guard-tracked-files.test.sh b/scripts/ci/tests/guard-tracked-files.test.sh index 681c3ad..40d0f34 100755 --- a/scripts/ci/tests/guard-tracked-files.test.sh +++ b/scripts/ci/tests/guard-tracked-files.test.sh @@ -10,6 +10,10 @@ set -u here=$(cd "$(dirname "$0")" && pwd) +# The suite runs the guard, or a modified or mutated copy of it (GUARD=), +# so it runs only where no container engine can be reached. +"$here/../no-docker.sh" --check >/dev/null 2>&1 || + exec "$here/../no-docker.sh" "$BASH" "$here/guard-tracked-files.test.sh" "$@" guard=${GUARD:-"$here/../guard-tracked-files.sh"} work=$(mktemp -d "${TMPDIR:-/tmp}/guard-tests.XXXXXX") || exit 2 trap 'rm -rf "$work"' EXIT diff --git a/scripts/ci/tests/guard-trivyignore.test.sh b/scripts/ci/tests/guard-trivyignore.test.sh index a34117c..2e117da 100755 --- a/scripts/ci/tests/guard-trivyignore.test.sh +++ b/scripts/ci/tests/guard-trivyignore.test.sh @@ -10,6 +10,10 @@ set -u here=$(cd "$(dirname "$0")" && pwd) +# The suite runs the guard, or a modified or mutated copy of it (GUARD=), +# so it runs only where no container engine can be reached. +"$here/../no-docker.sh" --check >/dev/null 2>&1 || + exec "$here/../no-docker.sh" "$BASH" "$here/guard-trivyignore.test.sh" "$@" guard=${GUARD:-"$here/../guard-trivyignore.sh"} real=$(git -C "$here" rev-parse --show-toplevel) || exit 2 work=$(mktemp -d "${TMPDIR:-/tmp}/guard-trivyignore-tests.XXXXXX") || exit 2 diff --git a/scripts/ci/tests/no-docker-aa-unit.py b/scripts/ci/tests/no-docker-aa-unit.py new file mode 100644 index 0000000..8fb7472 --- /dev/null +++ b/scripts/ci/tests/no-docker-aa-unit.py @@ -0,0 +1,675 @@ +"""Pure tests of scripts/ci/no-docker-aa.py (T-38 of no-docker.test.sh). + +Usage: python3 -I -B no-docker-aa-unit.py + +The helper's source and the two AppArmor templates are read once, then +every call that would open, list, read or write a file, read a link, run a +program, open a pidfd or send a signal is replaced by one that raises: the +operations run against a fake System and a fake control directory only, +never against this machine. Each assertion prints "ok " or +"FAIL "; the cases are P-01 to P-11, W (the helper calls the +tested functions) and M1 to M19 (mutated copies of the helper, each of +which must fail at least one assertion of its case). The last line is +"SUMMARY ok, failed". Exit 0 only if nothing failed; 2 if a +mutant cannot be made exactly. +""" + +import ast +import builtins +import errno +import hashlib +import os +import signal +import socket +import subprocess +import sys + +sys.dont_write_bytecode = True +SOURCE_PATH = sys.argv[1] +with open(SOURCE_PATH) as f: + SOURCE = f.read() +HERE = os.path.dirname(os.path.abspath(SOURCE_PATH)) +with open(os.path.join(HERE, "no-docker", "apparmor.profile"), "rb") as f: + PROD_BYTES = f.read() +with open(os.path.join(HERE, "tests", "no-docker", "apparmor-negative.profile"), "rb") as f: + NEG_BYTES = f.read() + + +def load(src): + ns = {"__name__": "aa_under_test", "__file__": SOURCE_PATH} + exec(compile(src, SOURCE_PATH, "exec"), ns) + ns["say"] = lambda text: None # the helper's messages are not results + return ns + + +class IOAttempt(AssertionError): + pass + + +def tripwire(name): + def raiser(*args, **kwargs): + raise IOAttempt(f"{name} called in a pure test") + return raiser + + +for mod, names in ((os, ("open", "listdir", "readlink", "stat", "lstat", "fstat", "scandir", "walk", "write", + "read", "link", "rename", "unlink", "fsync", "fchmod", "pidfd_open", "kill", "fork", + "pipe", "waitpid", "setns")), + (signal, ("pidfd_send_signal", "signal")), + (socket, ("socket", "socketpair")), + (subprocess, ("run", "Popen", "check_output", "call"))): + for name in names: + if hasattr(mod, name): + setattr(mod, name, tripwire(f"{mod.__name__}.{name}")) +builtins.open = tripwire("open") + +A = load(SOURCE) +NAME = "flowcrm-nd-" + "0123456789abcdef" * 2 +NEGNAME = "flowcrm-nd-neg-" + "fedcba9876543210" * 2 +HOST = 4026531836 +NSINO = 4026532269 +HASH = "a" * 64 +results = {"ok": 0, "fail": 0} + + +def emit(tag, good, text): + results["ok" if good else "fail"] += 1 + print(f"{'ok ' if good else 'FAIL'} {tag} {text}") + + +# --- fakes ------------------------------------------------------------------- + +def rec(text, uid=0, mode=0o444, regular=True): + return {"uid": uid, "mode": mode, "regular": regular, "text": text} + + +class FakeCtl: + def __init__(self, files=None, race=None): + self.files = dict(files or {}) + self.race = race # a record that appears as "ns" just before a link + self.writes = [] + + def lstat(self, name): + return True if name in self.files else None + + def record(self, name): + return self.files.get(name) + + def write_new(self, tmp, final, text, replace): + self.writes.append((final, text)) + if final == "ns" and self.race is not None: + self.files["ns"] = self.race + if final in self.files and not replace: + return False + self.files[final] = rec(text) + return True + + +def init_task(pid=900, state="Z (zombie)", threads="1", name="sh", ino=NSINO): + return (pid, ino, {"Name": name, "State": state, "Threads": threads, "NSpid": f"{pid}\t1"}) + + +class FakeSys: + def __init__(self, listed=(("enforce",),), hash_now=HASH, tasks=None, sweeps=None, events=None): + # The modes NAME is listed with, one entry per call (the last one + # repeats); "unreadable" makes the list raise. + self.lists = [x if x == "unreadable" else list(x) for x in listed] + self.hash_now = hash_now + self.tasks = [init_task()] if tasks is None else tasks + self.sweeps = sweeps if sweeps is not None else [[("/proc/1/task/1/attr/apparmor/current", "unconfined\n")]] + self.calls = events if events is not None else [] + self.removed = [] + self.added = [] + self.procs = {} + self.start = {} + self.inodes = {} + + def profile_list(self): + self.calls.append("list") + modes = self.lists[0] if len(self.lists) == 1 else self.lists.pop(0) + if modes == "unreadable": + raise OSError(errno.EACCES, "denied") + return "".join(f"{NAME} ({m})\n" for m in modes) + "foreign (enforce)\n" + + def kernel_hash(self, name): + return self.hash_now + + def host_pidns(self): + return HOST + + def pidns_tasks(self): + self.calls.append("p1") + return self.tasks + + def label_sweep(self): + self.calls.append("sweep") + return self.sweeps[0] if len(self.sweeps) == 1 else self.sweeps.pop(0) + + def remove(self, name): + self.calls.append("remove") + self.removed.append(name) + self.lists = [[]] + + def template_bytes(self, path): + return NEG_BYTES if path.endswith("apparmor-negative.profile") else PROD_BYTES + + def parser_add(self, text): + self.added.append(text) + self.lists = [["enforce"]] + + def process_table(self): + return self.procs + + def pidfd_open(self, pid): + self.calls.append(f"pidfd_open {pid}") + return 77 + + def start_time(self, pid): + self.calls.append("start_time") + return self.start.get(pid, 5000) + + def ns_inodes(self, pid): + self.calls.append("ns_inodes") + return self.inodes.get(pid, (NSINO, 4026532267)) + + def pidfd_signal(self, fd, sig): + self.calls.append(f"signal {int(sig)}") + + def close(self, fd): + pass + + def sleep(self, s): + pass + + def clock(self): + return 0.0 + + +def ok_ctl(state="entered spid=4242\n", ns=f"pid:{NSINO} mnt:4026532267\n", hash_=f"sha256 {HASH}\n"): + files = {} + if state is not None: + files["state"] = rec(state, uid=1001, mode=0o600) + if ns is not None: + files["ns"] = rec(ns) + if hash_ is not None: + files["hash"] = rec(hash_) + return FakeCtl(files) + + +def release(P, ctl, fs, name=NAME): + """"released" (removed), "kept" (refused, nothing removed) or what else + happened.""" + try: + P["release"](name, ctl, fs) + return "released" if fs.removed == [name] else "not removed" + except P["Refused"] as e: + return "kept" if not fs.removed else f"removed and refused ({e})" + + +# --- the cases --------------------------------------------------------------- + +def p01(P, emit): + ok = P["name_ok"] + emit("P-01", ok("load", NAME) and ok("release", NAME), "a valid NAME is accepted by load and release") + emit("P-01", ok("load-negative-control", NEGNAME) and ok("release", NEGNAME) and ok("diag", NEGNAME), + "a valid NEGNAME is accepted by load-negative-control, release and diag") + emit("P-01", not ok("load", NEGNAME), "rejected: NEGNAME by load") + emit("P-01", not ok("load-negative-control", NAME), "rejected: NAME by load-negative-control") + ops = ("load", "load-negative-control", "release", "diag") + emit("P-01", not any(ok(o, NAME[:-1]) or ok(o, NEGNAME[:-1]) for o in ops), "rejected: 31 hex digits") + emit("P-01", not any(ok(o, NAME + "0") or ok(o, NEGNAME + "0") for o in ops), "rejected: 33 hex digits") + emit("P-01", not any(ok(o, NAME.upper()) or ok(o, NAME[:-1] + "A") for o in ops), "rejected: upper case") + bad = [NAME[:-2] + "/x", "../" + NAME, NAME + "\n", NAME + " x", "flowcrm-nd-" + "0" * 30 + "..", ""] + emit("P-01", not any(ok(o, b) for o in ops for b in bad), "rejected: path characters, a newline, a space, empty") + + +def p02(P, emit): + pl = P["parse_label"] + emit("P-02", pl(f"{NAME} (enforce)") == (NAME, "enforce"), "enforce: the name and its mode") + emit("P-02", pl(f"{NAME} (complain)") == (NAME, "complain"), "complain: the mode is complain") + emit("P-02", pl(f"other//&{NAME} (enforce)")[0] != NAME, "a stacked label is not the profile itself") + emit("P-02", pl("unconfined") == ("unconfined", None), "unconfined has no mode") + emit("P-02", pl("") is None and pl("\n") is None, "an empty label is nothing") + emit("P-02", pl(f"{NAME} (enforce)\n") == (NAME, "enforce"), "a trailing newline is ignored") + emit("P-02", pl(f":ns:{NAME} (enforce)")[0] != NAME, "a namespace-prefixed label is not the profile itself") + + +def p03(P, emit): + good = {"enabled": "Y", "mode": "enforce", "af_unix": True, "parser": True, "aa_exec": True} + + def passes(o): + return all(p for p, _ in P["preflight_findings"](o)) + + emit("P-03", passes(good), "enabled, enforce, af_unix and both tools pass") + for text, change in (("AppArmor disabled", {"enabled": "N"}), ("enabled unreadable", {"enabled": None}), + ("global mode complain", {"mode": "complain"}), ("global mode unreadable", {"mode": None}), + ("no af_unix class", {"af_unix": False}), ("af_unix unreadable", {"af_unix": None}), + ("apparmor_parser missing", {"parser": False}), ("aa-exec missing", {"aa_exec": False}), + ("a tool unreadable", {"parser": None})): + emit("P-03", not passes(dict(good, **change)), f"rejected: {text}") + emit("P-03", passes(dict(good, restrict_unconfined="1", restrict_userns="1")), + "the sysctl values are recorded, not gating") + emit("P-03", not passes({}), "rejected: an empty observation") + af = P["af_unix_feature"] + emit("P-03", af("file", "yes\n") and af("dir", "") and not af("file", "no\n") and not af("other", "") + and not af("missing", ""), "af_unix: a file reading yes or a directory counts; anything else does not") + + +def p04(P, emit): + lm = P["listed_modes"] + emit("P-04", lm(f"x (enforce)\n{NAME} (enforce)\n", NAME) == ["enforce"], "listed once in enforce mode") + emit("P-04", lm(f"{NAME} (enforce)\n{NAME} (enforce)\n", NAME) == ["enforce", "enforce"], "listed twice: both seen") + emit("P-04", lm(f"{NAME} (complain)\n", NAME) == ["complain"], "listed in complain mode") + fs = FakeSys(listed=(("enforce",),)) + try: + P["load"]("load", NAME, FakeCtl(), fs) + refused = False + except P["Refused"]: + refused = True + fs2 = FakeSys(listed=((),)) + P["load"]("load", NAME, FakeCtl(), fs2) + emit("P-04", lm("x (enforce)\n", NAME) == [] and refused and not fs.added and len(fs2.added) == 1, + "absent: loaded only then; a NAME listed already is never loaded again") + fs = FakeSys(listed=("unreadable",)) + try: + r = release(P, ok_ctl(), fs) + except OSError: + r = "error" + emit("P-04", r == "error" and not fs.removed, "an unreadable list is an error, nothing is removed") + emit("P-04", lm(f"{NAME}0 (enforce)\nx{NAME} (enforce)\n", NAME) == [], "similar foreign names are not the profile") + # The kernel hands the list out about a page per read (a seq_file). + listing = "".join(f"profile-{i:04d} (enforce)\n" for i in range(400)) + f"{NAME} (enforce)\n" + whole = P["read_all"](chunks(listing.encode(), 4096), P["LIST_MAX"]).decode() + try: + P["read_all"](chunks(b"x (enforce)\n" * 1000, 4096), 8192) + over = False + except OSError as e: + over = not P["gone"](e) + emit("P-04", NAME not in listing[:4096] and lm(whole, NAME) == ["enforce"] and over, + "the list is read to its end: NAME after its first 4096 bytes is found; a list over the limit is an error") + + +def chunks(data, size): + """A read(n) of data that gives at most size bytes a call, then b"".""" + pos = [0] + + def read(n): + part = data[pos[0]:pos[0] + min(n, size)] + pos[0] += len(part) + return part + return read + + +def p05(P, emit): + sr = P["sweep_result"] + lab = "/proc/9/task/9/attr/apparmor/current" + emit("P-05", sr([(lab, f"{NAME} (enforce)\n")], NAME)[0] == [lab], "a task carrying the profile is counted") + emit("P-05", sr([(lab, f"other//&{NAME} (enforce)\n")], NAME)[0] == [lab], "a stacked label is counted") + emit("P-05", sr([(lab, f":ns://{NAME} (enforce)\n")], NAME)[0] == [lab], "a prefixed label is counted") + emit("P-05", sr([(lab, f"{NAME}ff (enforce)\n")], NAME)[0] == [lab], "a longer name containing it is counted") + emit("P-05", sr([(lab, OSError(errno.ENOENT, "x")), (lab, OSError(errno.ESRCH, "x"))], NAME) == ([], []), + "a task that ended (ENOENT, ESRCH) is skipped") + emit("P-05", sr([(lab, OSError(errno.EACCES, "x"))], NAME)[1] != [], "an unreadable label (EACCES) is unknown") + emit("P-05", sr([(lab, OSError(errno.EINVAL, "x"))], NAME)[1] != [], "an unreadable label (EINVAL) is unknown") + seq = [[(lab, "unconfined\n")], [(lab, f"{NAME} (enforce)\n")]] + emit("P-05", P["sweeps_clear"](lambda: seq.pop(0), NAME) is not None, "two sweeps are required: a carrier in the second keeps it") + emit("P-05", P["sweeps_clear"](lambda: [(lab, "unconfined\n")], NAME) is None, "two clear sweeps pass") + # A stacked label longer than 4096 bytes, NAME after its first 4096. + long_label = "/".join(["x" * 200] * 24) + f"//&{NAME} (enforce)\n" + whole = P["label_from"](long_label.encode(), P["LABEL_MAX"]) + try: + P["label_from"](long_label[:4096].encode(), 4096) + cut = None + except OSError as e: + cut = e + emit("P-05", len(long_label) > 4096 and sr([(lab, whole)], NAME)[0] == [lab] and isinstance(cut, OSError) + and sr([(lab, cut)], NAME)[1] != [], + "a long stacked label read whole is counted; a read that fills its buffer (maybe cut) is unknown") + + +def p06(P, emit): + def case(text, want, ctl=None, fs=None, p1_called=None): + fs = fs or FakeSys() + r = release(P, ctl or ok_ctl(), fs) + good = r == want and (p1_called is None or ("p1" in fs.calls) == p1_called) + emit("P-06", good, f"{text}: {r}") + + case("init alive", "kept", fs=FakeSys(tasks=[init_task(state="S (sleeping)")])) + case("init a single-threaded zombie", "released") + case("no task in the namespace", "released", fs=FakeSys(tasks=[])) + case("state loading, no namespace recorded: no P1", "released", ctl=ok_ctl(state="loading\n", ns=None), p1_called=False) + case("state entering, namespace record missing", "kept", ctl=ok_ctl(state="entering\n", ns=None)) + ctl = ok_ctl() + ctl.files["ns"] = rec(f"pid:{NSINO} mnt:1\n", uid=1001) + case("namespace record not root-owned", "kept", ctl=ctl) + case("namespace record unparsable", "kept", ctl=ok_ctl(ns="pid:x\n")) + case("another task in the namespace", "kept", fs=FakeSys(tasks=[init_task(), (901, NSINO, {"Name": "sleep", "State": "S", "Threads": "1", "NSpid": "901\t7"})])) + case("the task scan unreadable (EACCES)", "kept", fs=FakeSys(tasks=[(5, OSError(errno.EACCES, "x"), None)])) + case("state missing or unknown: treated as entered", "kept", ctl=ok_ctl(state="garbage\n", ns=None)) + case("namespace record naming the host's namespace", "kept", ctl=ok_ctl(ns=f"pid:{HOST} mnt:1\n")) + case("state loading with a namespace recorded: treated as entered", "kept", + fs=FakeSys(tasks=[init_task(state="S (sleeping)")]), ctl=ok_ctl(state="loading\n"), p1_called=True) + fs = FakeSys(tasks=[init_task(state="S (sleeping)")]) + a = release(P, ok_ctl(state="not-forked\n", ns=None), FakeSys()) + b = release(P, ok_ctl(state="not-forked\n"), fs) + emit("P-06", a == "released" and b == "kept" and "p1" in fs.calls, + f"state not-forked: released without P1 only when no namespace is recorded ({a}, {b})") + # IC-1: the namespace record's format. + case("a record whose mount namespace is unknown (mnt:-)", "released", ctl=ok_ctl(ns=f"pid:{NSINO} mnt:-\n")) + bad = ["pid:0 mnt:1\n", "pid:-1 mnt:1\n", f"pid:{NSINO} mnt:1", f"pid:{NSINO} mnt:1\n\n", f"pid:0{NSINO} mnt:1\n", + f"pid:{NSINO} mnt:1\n", f"mnt:1 pid:{NSINO}\n", f"pid:{NSINO} mnt:1 x\n", f"pid:{NSINO}\n", + f"pid:{NSINO} mnt:\n", "pid:1e3 mnt:1\n", f"pid:{NSINO} mnt:-1\n", f"pid:+{NSINO} mnt:1\n", + "pid:99999999999999999999 mnt:1\n", f" pid:{NSINO} mnt:1\n", ""] + rejected = 0 + for text in bad: + fs = FakeSys(tasks=[]) + if release(P, ok_ctl(ns=text), fs) == "kept" and P["p0"](rec(text), HOST) is not None: + rejected += 1 + emit("P-06", rejected == len(bad), f"malformed namespace records are refused ({rejected} of {len(bad)} variants)") + + +def p07(P, emit): + rd, R = P["render"], P["Refused"] + sha = hashlib.sha256(PROD_BYTES).hexdigest() + + def refused(data, kind="production"): + try: + rd(data, kind, hashlib.sha256(data).hexdigest(), NAME) + return False + except R: + return True + + try: + rd(PROD_BYTES + b"\n", "production", sha, NAME) + mismatch = False + except R: + mismatch = True + emit("P-07", mismatch, "rejected: template bytes that differ from the pinned sha256") + text = rd(PROD_BYTES, "production", sha, NAME) + emit("P-07", "@" not in text and text.count(NAME) == 2, "every @NAME@ is replaced (profile name and ptrace peer)") + emit("P-07", " unix,\n" in text and not refused(PROD_BYTES), "the unix rule is accepted (not an exec mode)") + emit("P-07", refused(PROD_BYTES.replace(b" file,\n", b" file,\n /usr/bin/** px,\n")), "rejected: an exec transition") + emit("P-07", refused(PROD_BYTES.replace(b"flags=(enforce)", b"flags=(enforce,attach_disconnected)")), + "rejected: attach_disconnected in the production template") + emit("P-07", refused(PROD_BYTES.replace(b" signal,\n", b" signal,\n change_profile,\n")), "rejected: change_profile") + + +def p08(P, emit): + tf, R = P["template_for"], P["Refused"] + + def kind(op, name): + try: + return tf(op, name)[0] + except R: + return "refused" + + emit("P-08", kind("load-negative-control", NEGNAME) == "negative", "load-negative-control loads the negative template") + emit("P-08", kind("load", NAME) == "production" and kind("load", NEGNAME) == "refused", + "load never loads the negative template") + emit("P-08", kind("load-negative-control", NAME) == "refused", "the negative template needs NEGNAME") + fs = FakeSys(listed=((),)) + P["load"]("load", NAME, FakeCtl(), fs) + emit("P-08", len(fs.added) == 1 and "attach_disconnected" not in fs.added[0] and NAME in fs.added[0], + "load adds the production profile under NAME") + + +def p09(P, emit): + def case(text, want, fs, ctl=None): + r = release(P, ctl or ok_ctl(), fs) + emit("P-09", r == want, f"{text}: {r}") + + case("the profile vanished", "kept", FakeSys(listed=((),))) + case("its mode changed (complain)", "kept", FakeSys(listed=(("complain",),))) + case("its kernel hash changed", "kept", FakeSys(hash_now="b" * 64)) + case("unchanged", "released", FakeSys()) + case("no hash exported, at load or now", "released", FakeSys(hash_now=None), ok_ctl(hash_="sha256 none\n")) + fs = FakeSys(listed=((),)) + r = release(P, ok_ctl(state="loading\n", ns=None), fs) + emit("P-09", r == "not removed" and "remove" not in fs.calls and "sweep" not in fs.calls, + f"state loading and the name never loaded: success with nothing to remove ({r})") + + +def procs_tree(launcher=1000, sudo=1001, monitor=None, unshare=1002, init=1003, init_ns=NSINO, nspid=("1003", "1")): + sudo_argv = ["sudo", "-n", "unshare", "--net", "--mount", "--pid", "--fork", "--kill-child", "--mount-proc", + "--ipc", "--propagation", "private", "--", "/bin/sh", "-c", "INIT"] + p = {launcher: {"ppid": 999, "comm": "bash", "argv": ["bash"], "start": 1, "nspid": [str(launcher)], "pidns": HOST, "ruid": 1001}, + sudo: {"ppid": launcher, "comm": "sudo", "argv": sudo_argv, "start": 2, "nspid": [str(sudo)], "pidns": HOST, "ruid": 0}} + parent = sudo + if monitor: + p[monitor] = {"ppid": sudo, "comm": "sudo", "argv": sudo_argv, "start": 3, "nspid": [str(monitor)], "pidns": HOST, "ruid": 0} + parent = monitor + p[unshare] = {"ppid": parent, "comm": "unshare", "argv": sudo_argv[2:], "start": 4, "nspid": [str(unshare)], "pidns": HOST, "ruid": 0} + if init: + p[init] = {"ppid": unshare, "comm": "sh", "argv": ["/bin/sh", "-c", "INIT"], "start": 5000, "nspid": list(nspid), + "pidns": init_ns, "ruid": 0} + return p + + +def p10(P, emit): + fs = FakeSys() + fs.procs = procs_tree() + ctl = FakeCtl() + r = P["kill_init_once"](1000, ctl, fs) + order = [c for c in fs.calls if c.startswith(("pidfd_open", "start_time", "ns_inodes", "signal"))] + want = ["pidfd_open 1003", "start_time", "ns_inodes", "start_time", "signal 0", f"signal {int(signal.SIGKILL)}"] + recorded = ctl.writes == [("ns", f"pid:{NSINO} mnt:4026532267\n")] + emit("P-10", r == "killed" and order == want and recorded, + f"namespace record missing: held by a pidfd, start time checked around the namespace read, recorded, then killed ({order})") + fs = FakeSys() + fs.procs = procs_tree() + ctl = FakeCtl({"ns": rec(f"pid:{NSINO} mnt:4026532267\n")}) + r1 = P["kill_init_once"](1000, ctl, fs) + fs2 = FakeSys() + fs2.procs = procs_tree() + fs2.inodes = {1003: (NSINO + 1, None)} + ctl2 = FakeCtl({"ns": rec(f"pid:{NSINO} mnt:4026532267\n")}) + try: + P["kill_init_once"](1000, ctl2, fs2) + r2 = "killed" + except P["Refused"]: + r2 = "refused" + emit("P-10", r1 == "killed" and not ctl.writes and r2 == "refused" and f"signal {int(signal.SIGKILL)}" not in fs2.calls, + f"namespace recorded: nothing written; killed only in the recorded namespace ({r1}, {r2})") + fs = FakeSys() + fs.procs = procs_tree() + ctl = FakeCtl(race=rec(f"pid:{NSINO} mnt:4026532267\n")) + r1 = P["kill_init_once"](1000, ctl, fs) + fs2 = FakeSys() + fs2.procs = procs_tree() + ctl2 = FakeCtl(race=rec(f"pid:{NSINO + 5} mnt:-\n")) + try: + P["kill_init_once"](1000, ctl2, fs2) + r2 = "killed" + except P["Refused"]: + r2 = "refused" + emit("P-10", r1 == "killed" and r2 == "refused" and f"signal {int(signal.SIGKILL)}" not in fs2.calls, + f"pid 1 recorded its namespace first: the same one is accepted, another one stops the kill ({r1}, {r2})") + fs = FakeSys() + fs.procs = procs_tree() + fs.procs[1005] = dict(fs.procs[1001]) # a second sudo of the launcher: ambiguous + now = [0.0] + fs.clock = lambda: now[0] + + def tick(s_): + now[0] += 1.0 + fs.sleep = tick + try: + P["kill_init"](1000, FakeCtl(), fs) + r = "returned" + except P["Refused"]: + r = "refused" + emit("P-10", r == "refused" and f"signal {int(signal.SIGKILL)}" not in fs.calls and now[0] >= 8, + f"an ambiguous lookup is retried until the deadline, then refused without a signal ({r}, {now[0]:.0f}s)") + + +def p11(P, emit): + cf = P["ctldir_findings"] + path = "/home/runner/work/_temp/no-docker-ctl.Ab12Cd" + info = {"dir": True, "link": False, "uid": 1001, "mode": 0o700} + emit("P-11", cf(path, info, path, 1001, "runner", 1001) == [], "a valid CTLDIR is accepted") + bad = [("work/_temp/no-docker-ctl.Ab12Cd", info, None), + ("/home/runner/work/_temp/../_temp/no-docker-ctl.Ab12Cd", info, None), + (path, info, "/home/runner/work/_temp/real"), + (path, None, path), (path, dict(info, dir=False), path), (path, dict(info, link=True), path), + (path, dict(info, mode=0o755), path), (path, dict(info, uid=0), path), + ("/home/runner/work/x/no-docker-ctl.Ab12Cd", info, None), ("/home/runner/work/_temp/ctl.Ab12Cd", info, None), + ("/home/runner/work/_temp/no-docker-ctl.Ab", info, None)] + refused = sum(1 for p, i, real in bad if cf(p, i, real or p, 1001, "runner", 1001)) + user = sum(1 for args in ((1001, "root", 0), (1001, "other", 1002), (1001, "Runner", 1001), (1001, "", None)) + if cf(path, info, path, *args)) + emit("P-11", refused == len(bad) and user == 4, + f"CTLDIR refused: relative, not canonical, a link, missing, not a directory, mode, owner, place, name; SUDO_USER mismatch ({refused + user} of {len(bad) + 4})") + lf = P["launcher_findings"] + + def table(chain): + t = {1000: {"ppid": 999, "comm": "bash", "ruid": 1001}} + prev = 1000 + for i, comm in enumerate(chain): + t[2000 + i] = {"ppid": prev, "comm": comm, "ruid": 0} + prev = 2000 + i + return t, prev + + t, me = table(["sudo", "timeout", "python3"]) + t2, me2 = table(["sudo", "sudo", "timeout", "python3"]) + emit("P-11", lf(1000, t, me, 1001) == [] and lf(1000, t2, me2, 1001) == [], + "LAUNCHERPID accepted: the parent of the sudo that started this helper (with and without sudo's monitor)") + t3, me3 = table(["bash", "sudo", "timeout", "python3"]) + t4, me4 = table(["sudo", "a", "b", "c", "d", "timeout", "python3"]) + variants = [lf(1000, {**t, 1000: {"ppid": 999, "comm": "bash", "ruid": 0}}, me, 1001), + lf(1000, t3, me3, 1001), lf(4242, t, me, 1001), lf(1000, t4, me4, 1001), + lf(999, {**t, 999: {"ppid": 1, "comm": "bash", "ruid": 1001}}, me, 1001)] + emit("P-11", all(variants), "LAUNCHERPID refused: another uid, not the sudo's parent, absent, too far up, a grandparent") + fi = P["find_init"] + fake = procs_tree() + fake[1006] = dict(fake[1002], ppid=1003, pidns=NSINO) # a process inside named unshare, with its argv + emit("P-11", fi(procs_tree(), 1000, HOST, None) == ("found", 1003) + and fi(procs_tree(monitor=1010), 1000, HOST, NSINO) == ("found", 1003) + and fi(fake, 1000, HOST, None) == ("found", 1003), + "pid 1 is found below the launcher's sudo and unshare (with and without sudo's monitor; a process " + "inside disguised as unshare is not taken)") + two = procs_tree() + two[1004] = dict(two[1003]) + sudo2 = procs_tree() + sudo2[1005] = dict(sudo2[1001]) + argv = procs_tree() + argv[1001] = dict(argv[1001], argv=["sudo", "-n", "unshare", "--net", "--"]) + outcomes = [fi(procs_tree(init=None), 1000, HOST, None)[0], fi(argv, 1000, HOST, None)[0], + fi(sudo2, 1000, HOST, None)[0], fi(two, 1000, HOST, None)[0], + fi(procs_tree(init_ns=HOST), 1000, HOST, None)[0], + fi(procs_tree(nspid=("1003", "7")), 1000, HOST, None)[0], + fi(procs_tree(), 1000, HOST, NSINO + 1)[0], fi(procs_tree(), 4242, HOST, None)[0]] + emit("P-11", outcomes == ["none", "none", "ambiguous", "ambiguous", "none", "none", "ambiguous", "none"], + f"pid 1 is not taken: no child yet, another sudo argv, two sudos, two children, the host namespace, not pid 1, another recorded namespace, another launcher ({outcomes})") + + +CASES = [("P-01", p01), ("P-02", p02), ("P-03", p03), ("P-04", p04), ("P-05", p05), ("P-06", p06), ("P-07", p07), + ("P-08", p08), ("P-09", p09), ("P-10", p10), ("P-11", p11)] + + +def run_case(fn, ns, emit_fn): + try: + fn(ns, emit_fn) + except IOAttempt: + raise + except Exception as e: + emit_fn(fn.__name__.upper().replace("P", "P-", 1), False, f"raised {type(e).__name__}: {e}") + + +# --- W: the helper calls what is tested -------------------------------------- + +def calls_in(tree, fname): + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == fname: + return {n.func.id if isinstance(n.func, ast.Name) else n.func.attr + for n in ast.walk(node) if isinstance(n, ast.Call) + and isinstance(n.func, (ast.Name, ast.Attribute))} | \ + {n.id for n in ast.walk(node) if isinstance(n, ast.Name)} + return set() + + +def wiring(): + tree = ast.parse(SOURCE) + need = { + "main": {"preflight", "load", "kill_init", "release", "diag", "validated_ctldir", "launcher_findings", "name_ok"}, + "release": {"parse_state", "release_plan", "p0", "p1", "p2", "sweeps_clear", "listed_modes", "parse_ns"}, + "load": {"template_for", "render", "listed_modes"}, + "kill_init_once": {"find_init", "p0", "parse_ns", "ns_record"}, + "preflight": {"preflight_findings"}, + "sweeps_clear": {"sweep_result"}, + "profile_list": {"read_all"}, + "label_sweep": {"read_label"}, + "read_label": {"label_from"}, + } + for fname, names in need.items(): + missing = names - calls_in(tree, fname) + emit("W", not missing, f"{fname} uses {', '.join(sorted(names))}" + (f" (missing: {', '.join(sorted(missing))})" if missing else "")) + + +# --- M: mutants of the helper, each must fail its case ----------------------- + +MUTANTS = [ + ("M1", "P-05", "a sweep counting exact labels only", + " elif name in label:\n carriers.append(path)", " elif label.split(' (')[0] == name:\n carriers.append(path)"), + ("M2", "P-05", "an unreadable label counted as no carrier", + " if not gone(label):\n unknown.append", " if False:\n unknown.append"), + ("M3", "P-05", "a single sweep", "SWEEPS = 2", "SWEEPS = 1"), + ("M4", "P-06", "P1 skipped", + ' why = p1(sys_.pidns_tasks(), parse_ns(rec["text"])[0])\n if why:', + ' why = None\n if why:'), + ("M5", "P-09", "P2 skipped", + " why = p2(listed_modes(sys_.profile_list(), name), sys_.kernel_hash(name), hash_load)\n if why:", + " why = None\n if why:"), + ("M6", "P-09", "the kernel hash not compared", + ' if (hash_now or "none") != hash_load:', ' if False:'), + ("M7", "P-04", "a load that replaces a loaded profile", + ' if listed_modes(sys_.profile_list(), name):\n raise Refused(f"{name} is loaded already")', + ' if False:\n raise Refused(f"{name} is loaded already")'), + ("M8", "P-08", "the negative template reachable through load", + ' if op == "load" and name_ok("load", name):\n return ("production",)', + ' if op == "load" and name_ok("release", name):\n return ("negative" if "neg" in name else "production",)'), + ("M9", "P-01", "the name prefix not checked", + 'NAME_RE = re.compile(r"flowcrm-nd-[0-9a-f]{32}")', 'NAME_RE = re.compile(r"[a-z-]*[0-9a-f]{32}")'), + ("M10", "P-06", "an init that still runs accepted", + ' and st.get("State", "").startswith("Z"))', ' )'), + ("M11", "P-05", "a task that ended counted as unknown", + " return isinstance(e, OSError) and e.errno in (errno.ENOENT, errno.ESRCH)", " return False"), + ("M12", "P-05", "a stacked label not counted", + " elif name in label:\n carriers.append(path)", " elif label.startswith(name):\n carriers.append(path)"), + ("M13", "P-06", "a missing state treated as loading", + ' if text is None or STATE_RE.fullmatch(text) is None:\n return "unknown"', + ' if text is None or STATE_RE.fullmatch(text) is None:\n return "loading"'), + ("M14", "P-06", "a namespace record naming the host accepted", + " if pid_ino == host_ino:\n return \"the namespace record names the host's pid namespace\"", + " if False:\n return \"the namespace record names the host's pid namespace\""), + ("M15", "P-06", "state loading with a namespace recorded not escalated", + ' if state in ("loading", "not-forked") and not ns_exists:', ' if state == "loading" or (state == "not-forked" and not ns_exists):'), + ("M16", "P-06", "state not-forked accepted with a namespace recorded", + ' if state in ("loading", "not-forked") and not ns_exists:', ' if state == "not-forked" or (state == "loading" and not ns_exists):'), + ("M17", "P-10", "pid 1 signalled before its namespace is recorded", + " sys_.pidfd_signal(pidfd, 0)\n if pid_ino == host:", + " sys_.pidfd_signal(pidfd, 0)\n sys_.pidfd_signal(pidfd, signal.SIGKILL)\n if pid_ino == host:"), + ("M18", "P-04", "the profile list read once", + " chunk = read(1 << 16)\n if not chunk:\n return data", " return read(1 << 16)"), + ("M19", "P-05", "a label read that filled its buffer accepted", + " if len(data) >= limit:\n raise OSError(errno.EOVERFLOW", " if False:\n raise OSError(errno.EOVERFLOW"), +] + + +def mutants(): + for name, case, text, old, new in MUTANTS: + n = SOURCE.count(old) + if n != 1: + print(f"HARNESS ERROR: mutant {name} matches {n} times; the helper changed, the mutant must be updated") + sys.exit(2) + ns = load(SOURCE.replace(old, new)) + failed = [] + run_case(dict(CASES)[case], ns, lambda tag, good, t: None if good else failed.append(t)) + emit(name, bool(failed), f"{text}: {'rejected by ' + case if failed else 'NOT rejected by ' + case}") + + +for tag, fn in CASES: + run_case(fn, A, emit) +wiring() +mutants() +print(f"SUMMARY {results['ok']} ok, {results['fail']} failed") +sys.exit(0 if results["fail"] == 0 else 1) diff --git a/scripts/ci/tests/no-docker-probe-unit.py b/scripts/ci/tests/no-docker-probe-unit.py new file mode 100644 index 0000000..f477712 --- /dev/null +++ b/scripts/ci/tests/no-docker-probe-unit.py @@ -0,0 +1,464 @@ +"""Pure tests of the Linux rules of scripts/ci/no-docker-probe.py. + +Usage: python3 -I -B no-docker-probe-unit.py + +The probe's source is read once, then every call that would read /proc, +open a file or socket, run a program, fork or enter a namespace is replaced +by one that raises: the rules are tested on fixtures only, never on this +machine. Each assertion prints "ok " or "FAIL "; +the cases are C01 to C07 and C10 to C14 (the rules, the gate's label, +descriptor and fence checks and --root-exec's sanitation), W (the probe +calls the tested functions) and M1 to M21 (mutated copies of the probe, +each of which must fail at least one assertion of its case). The +last line is "SUMMARY ok, failed". Exit 0 only if nothing +failed; 2 if a mutant cannot be made exactly. +""" + +import ast +import builtins +import errno +import os +import resource +import signal +import socket +import subprocess +import sys + +sys.dont_write_bytecode = True +SOURCE_PATH = sys.argv[1] +with open(SOURCE_PATH) as f: + SOURCE = f.read() + + +def load(src): + ns = {"__name__": "probe_under_test", "__file__": SOURCE_PATH} + exec(compile(src, SOURCE_PATH, "exec"), ns) + return ns + + +class IOAttempt(AssertionError): + pass + + +def tripwire(name): + def raiser(*args, **kwargs): + raise IOAttempt(f"{name} called in a pure test") + return raiser + + +for mod, names in ((os, ("open", "listdir", "readlink", "getxattr", "setns", "fork", "stat", "lstat", "fstat", + "scandir", "walk", "pipe", "kill", "waitpid", "closerange", "close", "dup2", + "set_inheritable", "execv", "execvp", "access", "statvfs", "isatty")), + (signal, ("signal", "pthread_sigmask")), + (resource, ("getrlimit",)), + (socket, ("socket", "socketpair", "if_nameindex")), + (subprocess, ("run", "Popen", "check_output", "call"))): + for name in names: + if hasattr(mod, name): + setattr(mod, name, tripwire(f"{mod.__name__}.{name}")) +builtins.open = tripwire("open") + +P = load(SOURCE) +INIT = "pid:[4026531836]" +HOST_NS = f"net=net:[4026531833] mnt=mnt:[4026531832] pid={INIT} ipc=ipc:[4026531839]" +results = {"ok": 0, "fail": 0} + + +def emit(tag, good, text): + results["ok" if good else "fail"] += 1 + print(f"{'ok ' if good else 'FAIL'} {tag} {text}") + + +def passes(findings): + return bool(findings) and all(p for p, _ in findings) + + +def failing(findings): + return [t for p, t in findings if not p] + + +def status(**over): + st = {"Name": "python3", "Uid": "1001\t1001\t1001\t1001", "Gid": "1001\t1001\t1001\t1001", "Groups": "", + "NoNewPrivs": "1", "NSpid": "20", "CapInh": "0000000000000000", "CapPrm": "0000000000000000", + "CapEff": "0000000000000000", "CapBnd": "0000000000000000", "CapAmb": "0000000000000000"} + st.update(over) + return {k: v for k, v in st.items() if v is not None} + + +def mount(point, fstype, opts=("rw",), sup=("rw",)): + return {"id": hash(point) & 0xFFFF, "point": point, "fstype": fstype, "opts": list(opts), "super": list(sup)} + + +def valid_mounts(): + vis = {p: mount(p, t) for p, t in (("/tmp", "tmpfs"), ("/var/tmp", "tmpfs"), ("/dev/shm", "tmpfs"), + ("/run", "tmpfs"), ("/dev/mqueue", "mqueue"))} + vis["/proc"] = mount("/proc", "proc") + vis["/sys/fs/cgroup"] = mount("/sys/fs/cgroup", "tmpfs", ("ro",)) + vis["/"] = mount("/", "ext4", ("ro",)) + return vis + + +HOST_DEVS = {"/tmp": "2049", "/var/tmp": "2049", "/dev/shm": "27", "/run": "29", "/dev/mqueue": "23"} +DEVS = {"/tmp": "54", "/var/tmp": "55", "/dev/shm": "56", "/run": "53", "/dev/mqueue": "51"} +PROCS = [("1", {"Name": "sh", "Uid": "0\t0\t0\t0"}), ("20", status()), ("21", status(Name="bash"))] + + +def valid_obs(**over): + o = {"status": status(), "group": "runner", "uid": 1001, "pid": 20, "proc_self": "20", + "ns": {"net": "net:[4026532270]", "mnt": "mnt:[4026532267]", "pid": "pid:[4026532269]", + "ipc": "ipc:[4026532268]"}, + "host_ns": {"net": "net:[4026531833]", "mnt": "mnt:[4026531832]", "pid": INIT, "ipc": "ipc:[4026531839]"}, + "host_ns_raw": HOST_NS, "processes": list(PROCS), "sid": 17, "sid_ours": True, "tty": "ENXIO", + "stdio": [(0, "/dev/null"), (1, "pipe"), (2, "pipe")], "interfaces": ["lo"], "work": "/tmp/no-docker.x", + "mounts": valid_mounts(), "host_devs": dict(HOST_DEVS), "devs": dict(DEVS), "writable": [], + "cwd": "/home/runner/work", "cwd_reachable": True} + o.update(over) + return o + + +# --- the cases; each takes the probe namespace and an emit function -------- + +def c01(P, emit): + f = P["eval_pidns"]("20", 20, "pid:[4026532269]", HOST_NS, "20") + emit("C01", passes(f) and len(f) == 4, f"a private /proc with NSpid of one entry, this pid, passes ({failing(f)})") + f = P["evaluate_linux"](valid_obs()) + emit("C01", passes(f), f"the whole mechanism check passes on a valid observation ({failing(f)})") + + +def c02(P, emit): + variants = [ + ("NSpid of two entries (a /proc of an ancestor namespace)", ("4711\t20", 20, "pid:[4026532269]", HOST_NS, "20")), + ("NSpid of one entry that is not this pid", ("21", 20, "pid:[4026532269]", HOST_NS, "20")), + ("NSpid missing", ("", 20, "pid:[4026532269]", HOST_NS, "20")), + ("NSpid malformed", ("20 x", 20, "pid:[4026532269]", HOST_NS, "20")), + ("own pid namespace is the caller's", ("20", 20, INIT, HOST_NS, "20")), + ("caller not in the initial pid namespace", ("20", 20, "pid:[4026532269]", HOST_NS.replace(INIT, "pid:[4026532001]"), "20")), + ("own namespace is the initial one, the caller claimed another", ("20", 20, INIT, HOST_NS.replace(INIT, "pid:[4026532001]"), "20")), + ("caller given twice", ("20", 20, "pid:[4026532269]", HOST_NS + f" pid={INIT}", "20")), + ("caller a near miss", ("20", 20, "pid:[4026532269]", HOST_NS.replace(INIT, "pid:[04026531836]"), "20")), + ("caller missing", ("20", 20, "pid:[4026532269]", "net=net:[1]", "20")), + ("own namespace malformed", ("20", 20, "pid:[x]", HOST_NS, "20")), + ("/proc/self is another pid", ("20", 20, "pid:[4026532269]", HOST_NS, "4711")), + ] + for text, args in variants: + f = P["eval_pidns"](*args) + emit("C02", not passes(f), f"rejected: {text} ({len(failing(f))} failing)") + f = P["evaluate_linux"](valid_obs(ns=dict(valid_obs()["ns"], pid=INIT))) + emit("C02", not passes(f), "rejected by the whole check: a pid namespace shared with the caller") + + +def c03(P, emit): + ev = P["eval_processes"] + emit("C03", passes(ev(PROCS, 1001)), "our processes and the root shell as pid 1 pass") + emit("C03", passes(ev(PROCS + [("22", OSError(errno.ENOENT, "gone"))], 1001)), "a process that ended is skipped") + emit("C03", passes(ev(PROCS + [("22", OSError(errno.ESRCH, "gone"))], 1001)), "a process that ended (ESRCH) is skipped") + bad = [ + ("a process of another user", PROCS + [("30", status(Uid="0\t0\t0\t0", Name="sleep"))]), + ("a host daemon under our uid", PROCS + [("31", status(Name="dockerd"))]), + ("Runner.Worker visible", PROCS + [("32", status(Name="Runner.Worker"))]), + ("pid 1 is systemd", [("1", {"Name": "systemd", "Uid": "0\t0\t0\t0"})] + PROCS[1:]), + ("pid 1 is not root", [("1", {"Name": "sh", "Uid": "1001\t1001\t1001\t1001"})] + PROCS[1:]), + ("pid 1 missing", PROCS[1:]), + ("pid 1 unreadable (hidepid)", [("1", OSError(errno.EPERM, "hidden"))] + PROCS[1:]), + ("a status unreadable (EACCES)", PROCS + [("33", OSError(errno.EACCES, "denied"))]), + ("a process without uids", PROCS + [("34", {"Name": "x"})]), + ] + for text, entries in bad: + emit("C03", not passes(ev(entries, 1001)), f"rejected: {text}") + + +def c04(P, emit): + ev = P["eval_creds"] + emit("C04", passes(ev(status(), "runner")), "no capability, no_new_privs, not root passes") + for cap in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb"): + emit("C04", not passes(ev(status(**{cap: "0000000000000400"}), "runner")), f"rejected: {cap} not zero") + emit("C04", not passes(ev(status(**{cap: None}), "runner")), f"rejected: {cap} missing") + emit("C04", not passes(ev(status(CapEff="zzzz"), "runner")), "rejected: an unparsable capability set") + emit("C04", not passes(ev(status(NoNewPrivs="0"), "runner")), "rejected: no_new_privs unset") + emit("C04", not passes(ev(status(Uid="1001\t0\t1001\t1001"), "runner")), "rejected: a uid 0") + + +def c05(P, emit): + ev = P["eval_creds"] + for gid in ("1000", "1001"): + emit("C05", passes(ev(status(Gid="\t".join([gid] * 4)), "g")), f"primary gid {gid} passes") + for gid, why in (("999", "the last system gid"), ("118", "the docker group"), ("6", "disk"), ("0", "root")): + emit("C05", not passes(ev(status(Gid="\t".join([gid] * 4)), "g")), f"rejected: primary gid {gid} ({why})") + emit("C05", not passes(ev(status(Gid="1001\t1001\t118\t1001"), "g")), "rejected: gids that differ") + emit("C05", not passes(ev(status(Gid=None), "g")), "rejected: gids missing") + emit("C05", not passes(ev(status(Groups="118"), "g")), "rejected: a supplementary group") + + +def c06(P, emit): + ev = P["eval_mounts"] + emit("C06", passes(ev(valid_mounts(), HOST_DEVS, DEVS, "/w", [])), "private mounts, one procfs, masked cgroups pass") + v = valid_mounts() + v["/proc"]["super"] = ["rw", "hidepid=0"] + emit("C06", passes(ev(v, HOST_DEVS, DEVS, "/w", [])), "hidepid=0 passes") + for text, change in ( + ("a second procfs", lambda v: v.update({"/mnt/p": mount("/mnt/p", "proc")})), + ("an nsfs file", lambda v: v.update({"/x": mount("/x", "nsfs")})), + ("/proc with hidepid=2", lambda v: v["/proc"].update(super=["rw", "hidepid=2"])), + ("/proc with hidepid=invisible", lambda v: v["/proc"].update(super=["rw", "hidepid=invisible"])), + ("/proc with hidepid=noaccess", lambda v: v["/proc"].update(super=["rw", "hidepid=noaccess"])), + ("/proc with hidepid=ptraceable", lambda v: v["/proc"].update(super=["rw", "hidepid=ptraceable"])), + ("/proc with subset=pid", lambda v: v["/proc"].update(opts=["rw", "subset=pid"])), + ("/proc not a procfs", lambda v: v["/proc"].update(fstype="tmpfs")), + ("/proc missing", lambda v: v.pop("/proc")), + ("/tmp not private", lambda v: v["/tmp"].update(fstype="ext4")), + ("/sys/fs/cgroup writable", lambda v: v["/sys/fs/cgroup"].update(opts=["rw"]))): + v = valid_mounts() + change(v) + emit("C06", not passes(ev(v, HOST_DEVS, DEVS, "/w", [])), f"rejected: {text}") + emit("C06", not passes(ev(valid_mounts(), HOST_DEVS, dict(DEVS, **{"/tmp": "2049"}), "/w", [])), + "rejected: /tmp is the caller's device") + emit("C06", not passes(ev(valid_mounts(), HOST_DEVS, DEVS, "/w", ["/home"])), "rejected: a writable mount left") + + +def c07(P, emit): + ev = P["eval_stdio"] + for text, kinds in (("a standard descriptor is a Unix socket", [(0, "/dev/null"), (1, "unix socket"), (2, "pipe")]), + ("a standard descriptor is a TCP socket", [(0, "/dev/null"), (1, "pipe"), (2, "socket")])): + emit("C07", not passes(ev(kinds)), f"rejected: {text}") + emit("C07", passes(ev([(0, "/dev/null"), (1, "pipe"), (2, "pipe")])), "/dev/null and pipes pass") + for text, kinds in (("a terminal", [(0, "terminal"), (1, "pipe"), (2, "pipe")]), + ("a regular file", [(0, "/dev/null"), (1, "file"), (2, "pipe")]), + ("a descriptor that cannot be examined (EACCES)", [(0, "/dev/null"), (1, "pipe"), (2, "unreadable (EACCES)")]), + ("fd 0 closed", [(0, "closed"), (1, "pipe"), (2, "pipe")])): + emit("C07", not passes(ev(kinds)), f"rejected: {text}") + + +def c10(P, emit): + calls = [] + said = [] + + def say(p, t): + said.append(p) + + def after(): + calls.append(1) + + obs = lambda: valid_obs() + ok = P["gated"](obs, lambda o: [(True, "a"), (True, "b")], after, say) + emit("C10", ok and calls == [1], "with every finding passed the write and connect phase runs once") + for text, observe, evaluate in ( + ("a finding failed", obs, lambda o: [(True, "a"), (False, "b"), (True, "c")]), + ("no findings", obs, lambda o: []), + ("the evaluation raised ValueError", obs, lambda o: int("x")), + ("the evaluation raised KeyError", obs, lambda o: {}["k"]), + ("the observation raised OSError", lambda: (_ for _ in ()).throw(OSError(errno.EIO, "io")), lambda o: [(True, "a")]), + ("the whole check on an observation with a writable mount", lambda: valid_obs(writable=["/home"]), P["evaluate_linux"]), + ("the whole check on a terminal", lambda: valid_obs(tty="opened"), P["evaluate_linux"])): + calls.clear() + said.clear() + ok = P["gated"](observe, evaluate, after, say) + emit("C10", not ok and calls == [] and False in said, f"not entered when {text}") + + +NAME = "flowcrm-nd-" + "0123456789abcdef" * 2 +LABEL = f"{NAME} (enforce)\n" +ROOT_SH = {"Name": "sh", "Uid": "0\t0\t0\t0"} +OURS = {"Name": "bash", "Uid": "1001\t1001\t1001\t1001"} + + +def c11(P, emit): + ev = P["eval_label"] + emit("C11", passes(ev(LABEL, [], NAME)), "this process confined by exactly NAME in enforce mode passes") + emit("C11", passes(ev(LABEL, [("1", ROOT_SH, "unconfined\n"), ("21", OURS, LABEL)], NAME)), + "every other process confined by exactly NAME, pid 1 the root shell, passes") + for text, own, others in ( + ("a stacked label", f"other//&{NAME} (enforce)\n", []), + ("complain mode", f"{NAME} (complain)\n", []), + ("another process unconfined", LABEL, [("1", ROOT_SH, "unconfined\n"), ("21", OURS, "unconfined\n")]), + ("pid 1 unconfined but not the root shell", LABEL, [("1", OURS, "unconfined\n")]), + ("the label missing", OSError(errno.ENOENT, "missing"), [])): + emit("C11", not passes(ev(own, others, NAME)), f"rejected: {text}") + + +def fds(**over): + o = {"pid": 20, "listed_pid": 20, "fds": [(0, "/dev/null"), (1, "pipe"), (2, "pipe")]} + o.update(over) + return o + + +STD = [(0, "/dev/null"), (1, "pipe"), (2, "pipe")] + + +def c12(P, emit): + ev = P["eval_fds"] + emit("C12", passes(ev(fds())), "fd 0 /dev/null, fds 1 and 2 pipes, nothing else passes") + emit("C12", not passes(ev(fds(fds=STD + [(3, "unix socket")]))), "rejected: an extra fd 3 (a Unix socket)") + emit("C12", not passes(ev(fds(fds=STD + [(7, "file")]))), "rejected: an extra fd 7 (a file)") + emit("C12", passes(ev(fds(fds=STD + [(3, "closed")]))), "the listing's own descriptor (closed by now) is ignored") + emit("C12", not passes(ev(fds(fds=[(0, "/dev/null"), (1, "unix socket"), (2, "pipe")]))), "rejected: fd 1 a socket") + emit("C12", not passes(ev(fds(fds=[(0, "/dev/null"), (1, "pipe"), (2, "file")]))), "rejected: fd 2 a regular file") + emit("C12", not passes(ev(fds(fds=STD + [(5, "unreadable (EIO)")]))), "rejected: an fd that cannot be examined") + emit("C12", not passes(ev(fds(fds=[(0, "character device"), (1, "pipe"), (2, "pipe")]))), "rejected: fd 0 not /dev/null (1:3)") + emit("C12", not passes(ev(fds(listed_pid=4711))), "rejected: the listing of another process") + + +def fence(**over): + o = {"stream": {"sock": True, "uid": 1001, "dac": True, "result": "EACCES"}, + "dgram": {"sock": True, "uid": 1001, "dac": True, "result": "EACCES"}, "twin": "ok"} + for k, v in over.items(): + o[k] = dict(o[k], **v) if isinstance(v, dict) else v + return o + + +def c13(P, emit): + ev = P["eval_fence"] + emit("C13", passes(ev(fence(), 1001)), "both fence sockets refused with EACCES, the twin connects: passes") + for text, f in (("both fence sockets reached", fence(stream={"result": "ok"}, dgram={"result": "ok"})), + ("the stream connect refused (ECONNREFUSED)", fence(stream={"result": "ECONNREFUSED"})), + ("the stream fence missing (ENOENT)", fence(stream={"result": "ENOENT"})), + ("the stream connect denied with EPERM", fence(stream={"result": "EPERM"})), + ("the datagram timing out", fence(dgram={"result": "timeout"})), + ("file permissions not shown to allow the fence", fence(dgram={"dac": False})), + ("the datagram refused but the stream reached", fence(stream={"result": "ok"})), + ("the twin not connecting", fence(twin="ECONNREFUSED"))): + emit("C13", not passes(ev(f, 1001)), f"rejected: {text}") + + +def c14(P, emit): + ev = P["eval_root_fds"] + before = [(0, "pipe"), (1, "pipe"), (2, "pipe"), (3, "closed"), (5, "unix socket"), (6, "file")] + good, closed, _ = ev(before, [(0, "pipe"), (1, "pipe"), (2, "pipe")], "/dev/null") + emit("C14", good and closed == [5, 6], f"descriptors above 2 closed and none left: proceeds (closed {closed})") + emit("C14", ev(before, [(0, "pipe"), (1, "pipe"), (2, "pipe"), (5, "unix socket")], "/dev/null")[0] is False, + "rejected: a descriptor above 2 survives the close") + emit("C14", ev(before, [(0, "pipe"), (1, "pipe"), (2, "pipe"), (4, "closed")], "/dev/null")[0] is True, + "the listing's own descriptor (closed by now) is ignored") + emit("C14", ev(before, [(0, "pipe"), (1, "pipe"), (2, "pipe")], "terminal")[0] is False, "rejected: stdin not /dev/null") + + +CASES = [("C01", c01), ("C02", c02), ("C03", c03), ("C04", c04), ("C05", c05), ("C06", c06), ("C07", c07), + ("C10", c10), ("C11", c11), ("C12", c12), ("C13", c13), ("C14", c14)] + + +def run_case(fn, ns, emit_fn): + try: + fn(ns, emit_fn) + except IOAttempt: + raise + except Exception as e: + emit_fn(fn.__name__.upper(), False, f"raised {type(e).__name__}: {e}") + + +# --- W: the probe calls what is tested ------------------------------------- + +def calls_in(tree, fname): + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == fname: + return {n.func.id if isinstance(n.func, ast.Name) else n.func.attr + for n in ast.walk(node) if isinstance(n, ast.Call) + and isinstance(n.func, (ast.Name, ast.Attribute))} | \ + {n.id for n in ast.walk(node) if isinstance(n, ast.Name)} + return set() + + +def lines_of(tree, fname, names): + """{name: first line} of the calls of names in function fname.""" + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == fname: + found = {} + for n in ast.walk(node): + if isinstance(n, ast.Call) and isinstance(n.func, (ast.Name, ast.Attribute)): + call = n.func.id if isinstance(n.func, ast.Name) else n.func.attr + if call in names: + found[call] = min(found.get(call, n.lineno), n.lineno) + return found + return {} + + +def main_mode(tree, flag, fname): + """True if main calls fname inside an if whose test names flag.""" + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "main": + for n in ast.walk(node): + if isinstance(n, ast.If) and flag in {c.value for c in ast.walk(n.test) if isinstance(c, ast.Constant)}: + called = {c.func.id if isinstance(c.func, ast.Name) else c.func.attr + for b in n.body for c in ast.walk(b) + if isinstance(c, ast.Call) and isinstance(c.func, (ast.Name, ast.Attribute))} + if fname in called: + return True + return False + + +def wiring(): + tree = ast.parse(SOURCE) + need = [ + ("the gate evaluates the label (G3) and the descriptor table (G8)", "gate_exec", {"evaluate_gate", "eval_fds", "observe_fds"}), + ("the gate's evaluation goes through gated", "gate_exec", {"gated"}), + ("the gate checks the fence (G4, G6)", "gate_exec", {"fence_check", "eval_fence"}), + ("the gate checks the twin (G5)", "gate_exec", {"twin_check"}), + ("--root-exec closes and verifies the descriptors", "root_exec", {"closerange", "list_fds", "eval_root_fds", "dup2", "execv"}), + ("the gate runs the kept after-gate checks and the engine check (endpoints)", "gate_exec", {"after_gate_linux", "engine_check"}), + ] + for text, fname, names in need: + missing = names - calls_in(tree, fname) + emit("W", not missing, f"{text}: {fname} uses {', '.join(sorted(names))}" + (f" (missing: {', '.join(sorted(missing))})" if missing else "")) + for flag, fname in (("--gate-exec", "gate_exec"), ("--root-exec", "root_exec"), ("--seal", "seal")): + emit("W", main_mode(tree, flag, fname), f"main runs {fname} for {flag}") + ok_check = {"check_mechanism_linux", "engine_check"} <= calls_in(tree, "main") + emit("W", ok_check, "main runs check_mechanism_linux and then engine_check without arguments (--check)") + nested = calls_in(tree, "check_mechanism_linux") + emit("W", {"gated", "evaluate_linux", "eval_label", "after_gate_linux"} <= nested and "endpoints" in calls_in(tree, "engine_check"), + "a nested run gates on G1 (evaluate_linux), G3 (eval_label) and G7 (after_gate_linux, then engine_check with endpoints)") + at = lines_of(tree, "gate_exec", {"signal", "pthread_sigmask", "execvp", "eval_fds"}) + emit("W", len(at) == 4 and at["eval_fds"] < at["signal"] < at["pthread_sigmask"] < at["execvp"], + f"the gate checks the descriptors, then resets every signal and clears the mask, then execs ({at})") + + +# --- M: mutants of the probe, each must fail its case ---------------------- + +MUTANTS = [ + ("M1", "C01", "the old NSpid rule (two entries or more)", + '(nspid.split() == [str(pid)], f"/proc shows', '(len(nspid.split()) >= 2, f"/proc shows'), + ("M4", "C05", "the primary group rule removed", + '(plain, f"the primary group is not a system group', '(True, f"the primary group is not a system group'), + ("M6", "C10", "the gate removed", + ' if not good:\n return False\n after()', ' after()\n if not good:\n return False'), + ("M11", "C03", "any unreadable process skipped", + ' if not gone(st):\n foreign.append', ' if False:\n foreign.append'), + ("M12", "C02", "the caller's pid namespace not checked", + '(caller == INIT_PID_NS, f"the caller is', '(True, f"the caller is'), + ("M13", "C02", "NSpid containing the pid accepted", + '(nspid.split() == [str(pid)], f"/proc shows', '(str(pid) in nspid.split(), f"/proc shows'), + ("M14", "C13", "any fence error accepted", + '(e.get("result") == "EACCES", f"{verb}', '(e.get("result") != "ok", f"{verb}'), + ("M15", "C11", "a stacked label accepted", + '(mine == want, f"this process is confined', '(mine.endswith(want), f"this process is confined'), + ("M16", "C12", "descriptors above 2 ignored", + 'for fd, k in sorted(kinds.items()) if fd >= 3 and k != "closed"]', 'for fd, k in sorted(kinds.items()) if False]'), + ("M17", "C13", "the file permission proof skipped", + '(e.get("dac") is True, f"file permissions allow', '(True, f"file permissions allow'), + ("M18", "C12", "a socket on fd 1 accepted", + '(all(kinds.get(fd) == want for fd, want in std),', '(all(kinds.get(fd) in (want, "unix socket") for fd, want in std),'), + ("M19", "C13", "the twin ignored", + '(o.get("twin") == "ok", f"a socket bound inside', '(True, f"a socket bound inside'), + ("M20", "C12", "a descriptor that cannot be examined taken as closed", + 'for fd, k in sorted(kinds.items()) if fd >= 3 and k != "closed"]', + 'for fd, k in sorted(kinds.items()) if fd >= 3 and k != "closed" and not k.startswith("unreadable")]'), + ("M21", "C12", "the listing of another process accepted", + '(o["listed_pid"] == o["pid"], f"the descriptors listed', '(True, f"the descriptors listed'), +] + + +def mutants(): + for name, case, text, old, new in MUTANTS: + n = SOURCE.count(old) + if n != 1: + print(f"HARNESS ERROR: mutant {name} matches {n} times; the probe changed, the mutant must be updated") + sys.exit(2) + ns = load(SOURCE.replace(old, new)) + failed = [] + run_case(dict(CASES)[case], ns, lambda tag, good, t: None if good else failed.append(t)) + emit(name, bool(failed), f"{text}: {'rejected by ' + case if failed else 'NOT rejected by ' + case}") + + +for tag, fn in CASES: + run_case(fn, P, emit) +wiring() +mutants() +print(f"SUMMARY {results['ok']} ok, {results['fail']} failed") +sys.exit(0 if results["fail"] == 0 else 1) diff --git a/scripts/ci/tests/no-docker-static.py b/scripts/ci/tests/no-docker-static.py new file mode 100644 index 0000000..59dcdee --- /dev/null +++ b/scripts/ci/tests/no-docker-static.py @@ -0,0 +1,392 @@ +"""Static rules of the Linux test isolation (T-39 of no-docker.test.sh). + +Usage: python3 -I -B no-docker-static.py + +Each rule is checked twice: on the tracked files of the repository, which +must pass, and on the same files with a mutant fixture applied, which must +fail. The fixtures are strings in this file, never files of the +repository. Two more lines check the AppArmor templates themselves. Each +check prints "ok " or "FAIL "; the last line is +"SUMMARY ok, failed". Exit 0 only if nothing failed. + + S-02 no code reloads, replaces or removes AppArmor policy, changes its + mode or writes its configuration (apparmor_parser -r/-R, aa-teardown + and the like, a restart of the service, writes to /etc/apparmor.d or + /sys/module/apparmor/parameters); + S-02b attach_disconnected appears only in the negative-control template + and the files that check it; + S-03b no sudo -E or --preserve-env anywhere; in no-docker.test.sh sudo + runs only the reviewed test commands; + S-03 in scripts/ci/no-docker.sh, sudo runs only the seven reviewed + commands (the helper's operations with their time limits, the + namespaces, sudo -n true), and kill-init only where sudo may still + run (never after the normal end, which would cost its 8 s retry + on every launch); + S-04 in no-docker.sh, the AppArmor helper and aa-exec are used only + inside a Linux arm of its case statements, and no-docker-aa.py's + main() refuses any other platform; + S-05 the AppArmor templates are exactly the reviewed rule set (RULES + below), the negative-control one differing only by + attach_disconnected; + S-06 the host socket listing is gone: no --sockets, --sockets-fast, + NO_DOCKER_SOCKETS, fast_listing or enumerate_sockets; + S-07 the negative control (--negative-control, load-negative-control) + appears only in no-docker.sh, no-docker-aa.py, no-docker.test.sh + and the unit and static tests; + S-08 the sha256 values no-docker-aa.py pins for the two templates are + those of the template files; + S-09 in the cleanup function of no-docker.sh, a failed removal of the + fence directory (C3), the control directory or the run directory + (C6) is recorded as a cleanup failure, so the launch ends 125 and + says so (C7; T-40 runs that function against stubs). + +Code means every tracked file except Markdown; lines that are comments +(#, //, /* or *) are not code either. +""" + +import hashlib +import os +import re +import subprocess +import sys + +sys.dont_write_bytecode = True + +SELF = "scripts/ci/tests/no-docker-static.py" +LAUNCHER = "scripts/ci/no-docker.sh" +PROBE_UNIT = "scripts/ci/tests/no-docker-probe-unit.py" +TEST = "scripts/ci/tests/no-docker.test.sh" +AA = "scripts/ci/no-docker-aa.py" +AA_UNIT = "scripts/ci/tests/no-docker-aa-unit.py" +TEMPLATE = "scripts/ci/no-docker/apparmor.profile" +NEGATIVE = "scripts/ci/tests/no-docker/apparmor-negative.profile" + +results = {"ok": 0, "fail": 0} + + +def emit(tag, good, text): + results["ok" if good else "fail"] += 1 + print(f"{'ok ' if good else 'FAIL'} {tag} {text}") + + +def tracked(root): + out = subprocess.run(["git", "-C", root, "ls-files", "-z"], capture_output=True, check=True).stdout + return [p for p in out.decode().split("\0") if p] + + +def read_repo(root): + """{path: text} of every tracked regular file (text decoded leniently).""" + files = {} + for p in tracked(root): + full = os.path.join(root, p) + if os.path.islink(full) or not os.path.isfile(full): + continue + with open(full, "rb") as f: + files[p] = f.read().decode("utf-8", "replace") + return files + + +def comment(line): + s = line.lstrip() + return s.startswith(("#", "//", "/*", "*")) + + +def code_lines(files, exempt=()): + """(path, line number, line) of every code line outside exempt.""" + for p, text in sorted(files.items()): + if p.endswith(".md") or p in exempt: + continue + for n, line in enumerate(text.splitlines(), 1): + if not comment(line): + yield p, n, line + + +# --- S-02 --------------------------------------------------------------------- + +S02 = [ + re.compile(r"apparmor_parser\b[^\n]*[\s\"'](-[A-Za-z]*[rR][A-Za-z]*\b|--(replace|remove)\b)"), + re.compile(r"\baa-(teardown|complain|enforce|disable)\b"), + re.compile(r"\b(systemctl|service)\b[^\n]*\bapparmor\b"), +] +APPARMOR_CONFIG = re.compile(r"/etc/apparmor\.d|/sys/module/apparmor/parameters") +WRITE = re.compile(r">|\btee\b|\bcp\b|\bmv\b|\binstall\b|\bln\b|\brm\b|\bsed\s+-i|[\"'][wax]b?\+?[\"']|O_WRONLY|O_RDWR" + r"|O_CREAT|write_text|write_bytes") + + +def s02(files): + found = [] + for p, n, line in code_lines(files, exempt=(SELF, AA_UNIT)): + if any(r.search(line) for r in S02) or (APPARMOR_CONFIG.search(line) and WRITE.search(line)): + found.append(f"{p}:{n}") + return found + + +# --- S-02b -------------------------------------------------------------------- + +def s02b(files): + return [f"{p}:{n}" for p, n, line in code_lines(files, exempt=(SELF, AA, AA_UNIT, NEGATIVE)) + if "attach_disconnected" in line] + + +# --- S-03b -------------------------------------------------------------------- + +PRESERVE = re.compile(r"\bsudo\b(\s+-[A-Za-z]+)*\s+-[A-Za-z]*E[A-Za-z]*\b|--preserve-env\b") +SUDO_USE = re.compile(r"(^|[\s;|&(`])sudo\s") +TEST_SUDO = [ + re.compile(r"sudo -n (mount|umount|kill|unshare)\s"), + re.compile(r'sudo -n "\$nd"\s'), + re.compile(r"sudo -n cat /sys/kernel/security/apparmor/profiles\b"), + re.compile(r'sudo -n cat "?/proc/[^/\s]+/attr/apparmor/(current|exec)\b'), + re.compile(r'sudo -n timeout [0-9]+ /usr/bin/python3 -I "?[^\s"]*/no-docker-aa\.py"? (load|release|diag)\s'), +] + + +def s03b(files): + found = [f"{p}:{n}" for p, n, line in code_lines(files, exempt=(SELF,)) if PRESERVE.search(line)] + for p, n, line in code_lines({TEST: files.get(TEST, "")}): + for m in SUDO_USE.finditer(line): + rest = line[m.end() - len("sudo") - 1:].lstrip() + if not any(r.match(rest) for r in TEST_SUDO): + found.append(f"{p}:{n}") + return found + + +# --- S-03 --------------------------------------------------------------------- + +# Each shape is the whole command: what may follow is a redirection of its +# errors, the end of the line, a list operator or "&". +END = r"(\s+2>/dev/null)?\s*(\|\||&&|;|&\s*$|$)" +LAUNCHER_SUDO = [re.compile(r + END) for r in ( + r"sudo -n true", + r'sudo -n timeout 10 /usr/bin/python3 -I "\$aa" preflight', + r'sudo -n timeout 20 /usr/bin/python3 -I "\$aa" "\$load_op" "\$profile" "\$ctldir"', + r'sudo -n timeout 10 /usr/bin/python3 -I "\$aa" kill-init "\$\$" "\$ctldir"', + r'sudo -n timeout 30 /usr/bin/python3 -I "\$aa" release "\$profile" "\$ctldir"', + r'sudo -n timeout 10 /usr/bin/python3 -I "\$aa" diag "\$profile"', +)] + [re.compile(r"sudo -n unshare --net --mount --pid --fork --kill-child --mount-proc --ipc --propagation private -- \\$")] + + +def s03(files): + found = [] + for p, n, line in code_lines({LAUNCHER: files.get(LAUNCHER, "")}): + if re.match(r"\s*for \w+ in [^;]*; do$", line): + continue # a word list (the tools looked for), not a command + for m in SUDO_USE.finditer(line): + rest = line[m.end() - len("sudo") - 1:].lstrip() + if not any(r.match(rest) for r in LAUNCHER_SUDO): + found.append(f"{p}:{n}") + lines = files.get(LAUNCHER, "").splitlines() + for n, line in enumerate(lines): + if "kill-init" in line and not comment(line) and (n == 0 or lines[n - 1].strip() != "elif sudo_may_run; then"): + found.append(f"{LAUNCHER}:{n + 1} (kill-init not guarded by sudo_may_run)") + return found + + +# --- S-04 --------------------------------------------------------------------- + +AA_USE = re.compile(r'"\$aa"|no-docker-aa\.py|aa-exec|apparmor_parser') + + +def s04(files): + found = [] + inside = None # the indentation of the Linux label whose arm this is + for n, line in enumerate(files.get(LAUNCHER, "").splitlines(), 1): + label = re.fullmatch(r"( *)(Linux|Darwin|\*)\)", line) + if label and label.group(2) == "Linux": + inside = label.group(1) + elif inside is not None and (line == inside + " ;;" or (label and label.group(1) == inside)): + inside = None + if comment(line) or not AA_USE.search(line) or line.startswith('aa="$here/no-docker-aa.py"'): + continue + if inside is None: + found.append(f"{LAUNCHER}:{n}") + tree = files.get(AA, "") + m = re.search(r"\ndef main\(argv, sys_=None\):\n if not sys\.platform\.startswith\(\"linux\"\):\n" + r" say\(\"Linux only\"\)\n return 2\n", tree) + if not m: + found.append(f"{AA}: main() does not refuse other platforms first") + return found + + +# --- S-05 --------------------------------------------------------------------- + +RULES = ["file,", "unix,", "network,", "signal,", "ptrace (read, readby, trace, tracedby) peer=@NAME@,", + "deny capability,", "deny mount,", "deny umount,", "deny pivot_root,", "deny network vsock,"] +FLAGS = {"production": "enforce", "negative": "enforce,attach_disconnected"} +FORBIDDEN = [ + (re.compile(r"\bcomplain\b"), "complain"), + (re.compile(r"\bchange_profile\b"), "change_profile"), + (re.compile(r"\buserns\b"), "userns"), + (re.compile(r"(^|\s)#?include\b"), "include"), + (re.compile(r"\b[pPcCuU]i?x\b|\bix\b"), "an exec transition"), + (re.compile(r"\battach_disconnected\b"), "attach_disconnected"), +] + + +def normalised(text): + return [re.sub(r"\s+", " ", line.strip()) for line in text.splitlines() if line.strip()] + + +def grammar(text, kind): + """Findings of a template against the reviewed rule set; [] if exact.""" + want = ["abi ,", f"profile @NAME@ flags=({FLAGS[kind]}) {{"] + RULES + ["}"] + got = normalised(text) + found = [] + if got != want: + extra = [x for x in got if x not in want] + missing = [x for x in want if x not in got] + found.append(f"not the reviewed rule set (extra {extra}, missing {missing}" + + (", order differs" if not extra and not missing else "") + ")") + for r, what in FORBIDDEN: + if kind == "negative" and what == "attach_disconnected": + continue + if any(r.search(line) for line in got): + found.append(what) + return found + + +def s05(files): + return [f"{TEMPLATE}: {x}" for x in grammar(files.get(TEMPLATE, ""), "production")] + \ + [f"{NEGATIVE}: {x}" for x in grammar(files.get(NEGATIVE, ""), "negative")] + + +# --- S-06 --------------------------------------------------------------------- + +LISTING = re.compile(r"--sockets(-fast)?\b|\bNO_DOCKER_SOCKETS\b|\bfast_listing\b|\benumerate_sockets\b") + + +def s06(files): + return [f"{p}:{n}" for p, n, line in code_lines(files, exempt=(SELF, AA_UNIT, PROBE_UNIT)) if LISTING.search(line)] + + +# --- S-07 --------------------------------------------------------------------- + +NEGATIVE_USE = re.compile(r"(^|[^-\w])--negative-control\b|\bload-negative-control\b") +NEGATIVE_FILES = (LAUNCHER, AA, TEST, AA_UNIT, PROBE_UNIT) + + +def s07(files): + return [f"{p}:{n}" for p, n, line in code_lines(files, exempt=(SELF,) + NEGATIVE_FILES) if NEGATIVE_USE.search(line)] + + +# --- S-08 --------------------------------------------------------------------- + +def pinned(aa_text): + """{kind: sha256} as no-docker-aa.py pins them.""" + found = {} + for kind in ("production", "negative"): + m = re.search(r'"' + kind + r'": \(os\.path\.join\([^)]*\),\s*"([0-9a-f]{64})"\)', aa_text) + found[kind] = m.group(1) if m else None + return found + + +def s08(files): + pins = pinned(files.get(AA, "")) + found = [] + for kind, path in (("production", TEMPLATE), ("negative", NEGATIVE)): + actual = hashlib.sha256(files.get(path, "").encode()).hexdigest() + if pins[kind] != actual: + found.append(f"{AA} pins {pins[kind]} for {path}, which is {actual}") + return found + + +# --- S-09 --------------------------------------------------------------------- + +def s09(files): + """In the cleanup function of no-docker.sh, each removal of the fence, + control and run directories is the condition of an `if` whose next line + records a cleanup failure.""" + lines = files.get(LAUNCHER, "").splitlines() + start = next((n for n, line in enumerate(lines) if re.fullmatch(r"( *)cleanup\(\) \{", line)), None) + if start is None: + return [f"{LAUNCHER}: no cleanup function"] + end = next((n for n in range(start + 1, len(lines)) + if lines[n] == lines[start][:len(lines[start]) - len(lines[start].lstrip())] + "}"), len(lines)) + found = [] + for var in ("fencedir", "ctldir", "work"): + removals = [n for n in range(start, end) if not comment(lines[n]) and f'rm -rf "${var}"' in lines[n]] + if not removals: + found.append(f"{LAUNCHER}: cleanup does not remove ${var}") + for n in removals: + guarded = re.fullmatch(r'\s*(el)?if (\[ -n "\$' + var + r'" \] && )?! rm -rf "\$' + var + r'"; then', lines[n]) + if not (guarded and n + 1 < end and re.match(r'\s*fail="', lines[n + 1])): + found.append(f"{LAUNCHER}:{n + 1}") + return found + + +# --- the rules and their mutant fixtures -------------------------------------- + +def add(path, text): + return lambda files: dict(files, **{path: files.get(path, "") + text}) + + +def edit(path, old, new): + def change(files): + if files.get(path, "").count(old) != 1: + raise LookupError(f"the fixture for {path} does not match exactly once") + return dict(files, **{path: files[path].replace(old, new)}) + return change + + +RULE_LIST = [ + ("S-02", "no AppArmor reload, replace, remove, mode change or configuration write", s02, + add("scripts/ci/no-docker.sh", "\nsudo -n apparmor_parser -r -K -q \"$profile\"\n")), + ("S-02b", "attach_disconnected only in the negative-control template", s02b, + add(TEMPLATE, " # flags=(attach_disconnected)\n attach_disconnected,\n")), + ("S-03b", "no sudo -E or --preserve-env; sudo in no-docker.test.sh only as reviewed", s03b, + add(TEST, "\nsudo -n -E true\n")), + ("S-03", "sudo in no-docker.sh runs only the seven reviewed commands", s03, + add(LAUNCHER, "\n sudo -n timeout 20 /usr/bin/python3 -I \"$aa\" load \"$profile\" \"$ctldir\" --replace\n")), + ("S-04", "the AppArmor helper only in the Linux arms of no-docker.sh; the helper refuses other platforms", s04, + edit(LAUNCHER, " Darwin)\n command -v sandbox-exec", " Darwin)\n sudo -n timeout 10 /usr/bin/python3 -I \"$aa\" preflight\n" + " command -v sandbox-exec")), + ("S-05", "the templates are exactly the reviewed rule set", s05, + edit(TEMPLATE, " deny network vsock,\n", "")), + ("S-06", "no host socket listing is left", s06, + add("scripts/ci/no-docker-probe.py", "\ndef fast_listing(deadline, paths):\n return 0\n")), + ("S-07", "the negative control only in the files that implement and test it", s07, + add(".github/workflows/ci.yml", "\n - run: scripts/ci/no-docker.sh --negative-control true\n")), + ("S-08", "the template hashes pinned in no-docker-aa.py are those of the files", s08, + edit(NEGATIVE, " deny network vsock,\n", " deny network inet,\n")), + ("S-09", "a failed removal of the fence, control or run directory is a cleanup failure (125)", s09, + edit(LAUNCHER, ' if [ -n "$fencedir" ] && ! rm -rf "$fencedir"; then\n' + ' fail="$fail; the fence directory $fencedir could not be removed"\n fi\n', + ' [ -z "$fencedir" ] || rm -rf "$fencedir"\n')), +] + + +def check_rules(files): + for tag, text, rule, mutate in RULE_LIST: + found = rule(files) + emit(tag, not found, f"repository: {text}" + (f" (violations: {', '.join(found)})" if found else "")) + try: + caught = rule(mutate(files)) + except LookupError as e: + emit(tag, False, f"mutant fixture: {e}") + continue + emit(tag, bool(caught), f"mutant fixture is rejected ({', '.join(caught) or 'NOT rejected'})") + + +def check_templates(files): + prod = files.get(TEMPLATE, "") + neg = files.get(NEGATIVE, "") + found = grammar(prod, "production") + emit("S-05", not found and "@NAME@" in prod and "attach_disconnected" not in prod, + f"template: {TEMPLATE} is the production rule set with flags=(enforce) ({found or 'exact'})") + as_production = grammar(neg, "production") + emit("S-05", grammar(neg, "negative") == [] and len(as_production) == 2 + and as_production[1] == "attach_disconnected" + and normalised(neg.replace("flags=(enforce,attach_disconnected)", "flags=(enforce)")) == normalised(prod), + f"template: {NEGATIVE} differs from the production template only by attach_disconnected ({as_production})") + + +def main(): + files = read_repo(os.path.abspath(sys.argv[1])) + check_rules(files) + check_templates(files) + print(f"SUMMARY {results['ok']} ok, {results['fail']} failed") + return 0 if results["fail"] == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/tests/no-docker.test.sh b/scripts/ci/tests/no-docker.test.sh new file mode 100755 index 0000000..3832ff0 --- /dev/null +++ b/scripts/ci/tests/no-docker.test.sh @@ -0,0 +1,3291 @@ +#!/usr/bin/env bash +# +# Regression tests for scripts/ci/no-docker.sh and its use by +# tests/devenv/unit.sh: a mutated bin/devenv that runs Docker commands must +# be refused or neutralized, whether it is passed as DEVENV or edited in +# place, and whichever way it reaches the Docker CLI or the API; and the +# isolation's limits on writes, signals, processes, namespaces and supported +# platforms must hold. +# +# Nothing here can change anything in the Docker daemon of this machine: +# - the mutant's payload only reads (`docker version`, GET /version and +# /_ping); it never removes, prunes or creates anything. A pre-flight +# check refuses to run any case unless every Docker command of the +# payload is `version` and every API request a GET of /version or /_ping, +# and its sha256 is checked again before every mutant is made from it; +# - its target is a canary engine of this suite: a Unix socket that +# answers like Docker and records every request; two cases name the real +# socket paths and must be refused at the socket; +# - the harness without the containment (T-05) runs, with its canary, +# behind an outer guard that leaves only the canary reachable: on macOS +# a sandbox-exec profile denying every other Unix socket, TCP to +# 22/2375/2376 and beyond loopback, writes under $HOME and to this +# suite's helpers and starting programs outside the sandbox; on Linux +# no-docker.sh itself, whose probe runs before the canary is started +# inside it; +# - the harness under test (T-03, T-04) runs on Linux behind the same +# guard, where it must refuse because the canary is a reachable engine. +# On macOS it runs on the host itself, as a developer would run it: +# macOS applies a second, different sandbox profile inside a sandbox +# only if it is identical, so the harness's own isolation cannot be +# tested behind the guard. Should that isolation fail, the read-only +# payload would reach the real daemon, which the cases detect; +# - the helpers this suite runs outside any isolation (drive.sh, +# canary.py) and the payload are tracked files (no-docker/) pinned to +# reviewed sha256 values in this file; their copies are kept outside +# every directory the code under test can write, and checked against +# those values immediately before each such run (T-14, T-23); +# - the other cases run plain commands that only print, connect to this +# suite's own listeners, write marker files in directories of this +# suite (and in protected places, where the write must fail; a marker +# that was written anyway is removed), send signal 0 or TERM to this +# suite's own `sleep` canaries, start processes of their own that +# sleep and must be ended by no-docker.sh (any left over are killed by +# pid only while their arguments carry the case's marker), attach a +# debugger to or read the memory map of this suite's own canary copies +# of /bin/sleep, read a pseudo-terminal of their own, run a bounded +# chain of forks of their own inside the isolation (T-30), remove the +# marker file of a run of their own (T-33), or sleep; +# - T-36 and T-38 run pure tests of the probe's rules and of the AppArmor +# helper: fixtures only, with every call that would read /proc, open a +# file or socket, run a program, fork, send a signal or change a +# descriptor made to raise. +# - T-39 reads the tracked files and checks static rules on them and on +# mutant fixtures held in memory. +# - T-40 runs the Linux cleanup function of no-docker.sh, taken from the +# script, in a shell of its own against stubs of rm, sudo and chmod, +# on directories of this suite only; T-41 computes L-16's projection +# from fixed timings; T-42 copies files of fixture directories of this +# suite (the opt-in NO_DOCKER_TEST_EVIDENCE copy, below), some of them +# replaced by a link to another of its directories at a chosen moment; +# T-43 runs keep_stream on /bin/sh commands, into a directory of this +# suite it can and one it cannot write; T-44 runs L-15c's reporting +# with fixed verdicts; T-45 runs L-07's caller with a stub in place of +# no-docker.sh (a pseudo-terminal and a socket pair of its own); +# T-46 counts lines of fixture files of its own; T-47 gives T-07's +# verdict fixed statuses and outputs of its own. +# - the L-tests (Linux CI only) check the kernel-enforced isolation from +# outside: sockets of this suite (stream, datagram, abstract, behind a +# bind-mounted alias, in a directory that cannot be listed, bound late, +# passing a descriptor) must be refused inside and reachable outside; +# descriptors, signals, escapes, the negative control, the release of +# the profile with carriers of this suite (aa-exec ... sleep, ended by +# pid), TERM at chosen moments, SIGKILL of the launch's own unshare, +# output relaying and the cost of a launch (L-16, last). They use sudo +# only for: mount and umount of the alias, kill of that unshare, unshare +# of a listener's network namespace, reading the profile list and a +# task's AppArmor labels, and no-docker-aa.py load, release and diag of +# this suite's own profiles (L-13, and releasing what a case left +# loaded); the profiles a case records as left loaded are released on +# exit, after its carriers ended. +# This suite itself runs no docker command. +# +# T-06, T-08 to T-15, T-21, T-22 and T-24 to T-35 are macOS only, T-16 to +# T-18 and L-01 to L-18 Linux only (INFO or SKIP on the other platform; on +# Linux CI a missing prerequisite is a failure); T-19, +# T-20, T-23, T-36, T-38, T-39, T-40, T-41, T-42, T-43, T-44, T-45, T-46 and T-47 are static or pure and run on both. Each case says +# how it fails against the no-docker.sh of 38b5ebd, dfb6ee4, e8ed7ed or +# 85cfb37, which these cases were written to catch. +# +# Usage: no-docker.test.sh +# HARNESS= checks another copy of tests/devenv/unit.sh instead of the +# current one, with every case behind the outer guard (it is unreviewed +# code); a harness without the containment must fail T-03 and T-04. On +# macOS a harness with it fails them too, as it cannot nest its sandbox. + +# shellcheck disable=SC2016 +# Single-quoted $... in sed and grep patterns and in the payloads stay literal. +# shellcheck disable=SC2319 +# "; result NAME $?" records the status of that condition on purpose. + +set -u + +here=$(cd -P "$(dirname "$0")" && pwd -P) +root=$(cd -P "$here/../../.." && pwd -P) +nd=$root/scripts/ci/no-docker.sh +os=$(uname -s) +# Opt-in, off by default: NO_DOCKER_TEST_EVIDENCE names an empty directory +# of this user, mode 0700, by its canonical physical path (absolute, no +# component of it a symbolic link: on macOS /private/var/..., not +# /var/...). At the end, before its directories are removed, the suite +# copies its case outputs there ($work, and the --work directories under +# $vt, whose content the commands wrote and which is data only): regular +# files with safe names and a single link only, each at most 16 MiB and +# 128 MiB in all, with MANIFEST.sha256 and SKIPPED.txt. Both trees are +# walked by directory descriptors, each directory opened from its already +# opened parent with O_NOFOLLOW (the destination and each source from / +# down), so no link is followed and a directory replaced meanwhile is +# skipped, not read through; files are read and written relative to their +# opened directory. The sources are given by their canonical physical paths +# too ($work and $vt are made physical when they are created); a source +# path is never resolved, and one with a link in it is refused (2) before +# anything is written. A copy that cannot be completed makes the suite +# fail (2). T-42 checks the copy. The cases keep what they would otherwise +# only filter or discard there: the checker outputs of T-19, T-36, T-38 +# and T-39 and the streams of the launches of T-01, T-02, T-07, T-16 (with +# its socket records), T-17, T-18 and L-16; a copy through tee that fails +# makes the suite fail (keep_stream, T-43). +# shellcheck disable=SC2016 # Python, not shell +keep_py=' +import errno, hashlib, os, re, stat, sys +SAFE = re.compile(r"[A-Za-z0-9._@+=,-]+") +FILE_MAX, TOTAL_MAX, DEPTH_MAX = 16 << 20, 128 << 20, 32 +DIR = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW +GONE = (errno.ELOOP, errno.ENOTDIR, errno.ENOENT, errno.ENXIO, errno.EOPNOTSUPP) +def refuse(text): + print(f"HARNESS ERROR: NO_DOCKER_TEST_EVIDENCE {text}", file=sys.stderr) + sys.exit(2) +def opendir(path): + # An absolute canonical path, opened one component at a time from / + # without following a link: a symbolic link anywhere in it fails. + if not os.path.isabs(path) or os.path.normpath(path) != path or path.startswith("//"): + raise OSError(errno.EINVAL, "not an absolute, canonical path") + fd = os.open("/", DIR) + try: + for part in path.split("/")[1:]: + if part: + nfd = os.open(part, DIR, dir_fd=fd) + os.close(fd) + fd = nfd + except BaseException: + os.close(fd) + raise + return fd +def check(dest): + try: + fd = opendir(dest) + except OSError as e: + refuse(f"{dest!r} is not the canonical physical path of a directory ({e.strerror})") + st = os.fstat(fd) + if stat.S_IMODE(st.st_mode) != 0o700 or st.st_uid != os.getuid(): + refuse(f"{dest} is not a directory of mode 0700 owned by this user") + if os.listdir(fd): + refuse(f"{dest} is not empty") + return fd +def create(dfd, name, data): + fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=dfd) + try: + view = memoryview(data) + while view: + view = view[os.write(fd, view):] + finally: + os.close(fd) +class Out: + # A directory of the copy: made (0700) and opened in its opened parent + # when the first file is kept in it. + def __init__(self, parent, name, fd=None): + self.parent, self.name, self.fd = parent, name, fd + def get(self): + if self.fd is None: + pfd = self.parent.get() + os.mkdir(self.name, 0o700, dir_fd=pfd) + self.fd = os.open(self.name, DIR, dir_fd=pfd) + return self.fd + def close(self): + if self.parent is not None and self.fd is not None: + os.close(self.fd) + self.fd = None +def copy(dest, pairs): + top = Out(None, None, check(dest)) + sums, skipped, failed, roots, names, kept = [], [], [], [], [], [0] + def gone(relp, e, what): + if e.errno in GONE: + skipped.append(f"{relp}: {errno.errorcode[e.errno]} ({what} replaced or removed meanwhile, not followed)") + else: + failed.append(f"{relp}: {e.strerror}") + def read(sfd, n, relp): + try: + fd = os.open(n, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=sfd) + except OSError as e: + gone(relp, e, "file") + return None + try: + st = os.fstat(fd) + if not stat.S_ISREG(st.st_mode): + skipped.append(f"{relp}: not a regular file") + return None + if st.st_nlink != 1: + skipped.append(f"{relp}: {st.st_nlink} hard links") + return None + parts, size = [], 0 + while size <= FILE_MAX: + b = os.read(fd, 1 << 20) + if not b: + break + parts.append(b) + size += len(b) + finally: + os.close(fd) + if size > FILE_MAX or kept[0] + size > TOTAL_MAX: + failed.append(f"{relp}: too large ({size} bytes, {kept[0]} kept)") + return None + kept[0] += size + return b"".join(parts) + def walk(sfd, rel, out, depth): + try: + entries = sorted(os.listdir(sfd)) + except OSError as e: + failed.append(f"{rel}: {e.strerror}") + return + for n in entries: + relp = f"{rel}/{n}" + if not SAFE.fullmatch(n): + skipped.append(f"{relp!r}: unsafe name") + continue + try: + st = os.stat(n, dir_fd=sfd, follow_symlinks=False) + except OSError as e: + gone(relp, e, "entry") + continue + if stat.S_ISDIR(st.st_mode): + if depth >= DEPTH_MAX: + failed.append(f"{relp}: deeper than {DEPTH_MAX} directories") + continue + try: + cfd = os.open(n, DIR, dir_fd=sfd) + except OSError as e: + gone(relp, e, "directory") + continue + sub = Out(out, n) + try: + walk(cfd, relp, sub, depth + 1) + finally: + os.close(cfd) + sub.close() + elif stat.S_ISREG(st.st_mode): + data = read(sfd, n, relp) + if data is not None: + create(out.get(), n, data) + sums.append((relp, hashlib.sha256(data).hexdigest())) + else: + what = "symbolic link, not followed" if stat.S_ISLNK(st.st_mode) else "not a regular file" + skipped.append(f"{relp}: {what}") + # Every source by its canonical physical path, opened like the + # destination before anything is written; one inside an earlier one is + # copied with it. + opened = [] + for pair in pairs: + name, _, src = pair.partition("=") + if not SAFE.fullmatch(name) or name in names or name in ("MANIFEST.sha256", "SKIPPED.txt"): + continue + try: + sfd = opendir(src) + except OSError as e: + refuse(f"source {name} {src!r} is not the canonical physical path of a directory ({e.strerror})") + if any(src == r or src.startswith(r + "/") for r in roots): + os.close(sfd) + continue + names.append(name) + roots.append(src) + opened.append((name, sfd)) + for name, sfd in opened: + out = Out(top, name) + try: + walk(sfd, name, out, 0) + finally: + os.close(sfd) + out.close() + create(top.fd, "MANIFEST.sha256", "".join(f"{h} {p}\n" for p, h in sorted(sums)).encode()) + create(top.fd, "SKIPPED.txt", "".join(s + "\n" for s in skipped + [f"FAILED {x}" for x in failed]).encode()) + if failed: + print("HARNESS ERROR: case outputs not kept: " + "; ".join(failed[:5]), file=sys.stderr) + sys.exit(1) +if sys.argv[1] == "check": + check(sys.argv[2]) +else: + copy(sys.argv[2], sys.argv[3:]) +' +keep=${NO_DOCKER_TEST_EVIDENCE-} +[ -z "$keep" ] || python3 -I -c "$keep_py" check "$keep" || exit 2 +work=$(mktemp -d "${TMPDIR:-/tmp}/no-docker-tests.XXXXXX") || exit 2 +work=$(cd -P "$work" && pwd -P) +[ -n "$work" ] && [ -d "$work" ] || exit 2 +# Read-only material (helpers, payload, harness trees): outside every +# directory code under test may write. On Linux it must also be visible +# inside the isolation, which hides /tmp and /var/tmp: $RUNNER_TEMP is +# mounted read-only there. +if [ "$os" = Linux ] && [ -n "${RUNNER_TEMP-}" ]; then + ro=$(mktemp -d "$RUNNER_TEMP/no-docker-tests-ro.XXXXXX") || exit 2 +else + ro=$work/ro + mkdir "$ro" || exit 2 +fi +ro=$(cd -P "$ro" && pwd -P) +helpers=$ro/helpers +mkdir "$helpers" || exit 2 +# Directories for --work: fresh and empty; on Linux under /var/tmp. +if [ "$os" = Linux ]; then + vt=$(mktemp -d /var/tmp/no-docker-tests.XXXXXX) || exit 2 +else + vt=$work/w + mkdir -m 700 "$vt" || exit 2 +fi +vt=$(cd -P "$vt" && pwd -P) +home=$(cd -P "$HOME" && pwd -P) +[ -n "$home" ] && [ -d "$home" ] || exit 2 +nonce=$$-$(date +%s) +home_marker=$home/.flowcrm-no-docker-test-$nonce +tmp=$(cd -P "${TMPDIR:-/tmp}" && pwd -P) +[ -n "$tmp" ] && [ "$tmp" != / ] || exit 2 +# macOS: the per-user temporary directory no-docker.sh uses (its markers). +ndtmp=$tmp +[ "$os" != Darwin ] || [ -n "${TMPDIR-}" ] || ndtmp=$(cd -P "$(getconf DARWIN_USER_TEMP_DIR)" && pwd -P) || exit 2 +# Next to $TMPDIR: writable by the user, outside what the sandbox allows. +outside_marker=$(dirname "$tmp")/.flowcrm-no-docker-test-$nonce +tmp_marker=/private/tmp/.flowcrm-no-docker-test-$nonce +# In $TMPDIR itself, outside the run's directory. +t11_marker=$tmp/.flowcrm-no-docker-test-t11-$nonce +cleanup() { + rm -f "$home_marker" "$outside_marker" "$tmp_marker" "$t11_marker" + # T-21's canaries in the shared directories, only while they are ours. + [ -z "${t21a_ino-}" ] || python3 -I -c "$t21_file" remove "$t21a" "$t21a_dev" "$t21a_ino" + [ -z "${t21b_ino-}" ] || python3 -I -c "$t21_file" remove "$t21b" "$t21b_dev" "$t21b_ino" + # The L-tests' listeners and carriers (this suite's children, by their + # recorded pid), the alias mount, and any profile a case left loaded. + [ ! -f "$work/l-listeners" ] || while read -r f; do + p=$(cut -d' ' -f1 "$f" 2>/dev/null) + case $p in "" | *[!0-9]*) ;; *) kill "$p" 2>/dev/null ;; esac + done <"$work/l-listeners" + [ ! -f "$work/l-carriers" ] || while read -r p; do + case $p in "" | *[!0-9]*) ;; *) kill "$p" 2>/dev/null && wait "$p" 2>/dev/null ;; esac + done <"$work/l-carriers" + [ -z "${l02_mounted-}" ] || sudo -n umount "$ro/l02alias" + [ ! -f "$work/l-release" ] || while read -r n c; do + if [ -d "$c" ]; then + sudo -n timeout 30 /usr/bin/python3 -I "$root/scripts/ci/no-docker-aa.py" release "$n" "$c" >/dev/null 2>&1 && rm -rf "$c" + fi + done <"$work/l-release" + chmod 700 "$ro/l03n1" "$ro/l03n2" "$ro/l03n0" 2>/dev/null + # Opt-in (NO_DOCKER_TEST_EVIDENCE): keep the case outputs first. + if [ -n "$keep" ] && ! python3 -I -c "$keep_py" copy "$keep" "work=$work" "vt=$vt"; then + rm -rf "$work" "$ro" "$vt" + exit 2 + fi + rm -rf "$work" "$ro" "$vt" +} +trap cleanup EXIT +pass=0 +fail=0 + +lcount=0 +result() { # name ok(0|1) [detail] + case $1 in L-*) lcount=$((lcount + 1)) ;; esac + if [ "$2" -eq 0 ]; then + pass=$((pass + 1)) + printf 'PASS %s\n' "$1" + else + fail=$((fail + 1)) + printf 'FAIL %s%s\n' "$1" "${3:+ $3}" + fi +} + +sha() { # file: its sha256 + { if command -v sha256sum >/dev/null 2>&1; then sha256sum; else shasum -a 256; fi; } <"$1" | cut -c1-64 +} + +# Files that code under test could write are read only when they are +# regular files, so a planted FIFO or link cannot hang or redirect a read. +regular() { [ -f "$1" ] && [ ! -L "$1" ]; } +count_in() { # pattern file: number of matching lines, 0 if not a regular file + local n=0 + regular "$2" && n=$(grep -c -- "$1" <"$2" 2>/dev/null) + echo "${n:-0}" +} + +new_work() { # a fresh, empty directory for --work + mktemp -d "$vt/w.XXXXXX" +} + +# The Linux launches of this suite, counted for L-16's projection of the +# guard job (a file: some run in subshells). A launch is counted when it is +# started, so one of L-15c that TERM ends before it made anything counts +# too: the projection can only come out higher, never lower. Refusals +# before any resource (T-18, L-07) are not counted. +count_launch() { printf x >>"$work/launch-count"; } +ndl() { # no-docker.sh, counted on Linux + [ "$os" != Linux ] || count_launch + "$nd" "$@" +} +# keep_stream MODE FILE [OTHER] COMMAND...: run COMMAND, a launch whose +# output a case reads, with that output copied by tee into FILE as the case +# reads it, and set ks_rc to COMMAND's own status: +# both stdout and stderr through tee into FILE +# out stdout through tee into FILE, stderr into the file OTHER +# err stderr through tee into FILE, stdout into the file OTHER +# COMMAND still writes to a pipe. Both statuses of the pipeline are taken at +# once; a copy that failed (tee's status not 0) is a harness error: the +# suite ends 2, also when COMMAND ended as the case expects. T-43. +keep_stream() { + local mode=$1 file=$2 other="" st + shift 2 + case $mode in out | err) other=$1 && shift ;; esac + case $mode in + both) + "$@" 2>&1 | tee "$file" >/dev/null + st=("${PIPESTATUS[@]}") + ;; + out) + "$@" 2>"$other" | tee "$file" >/dev/null + st=("${PIPESTATUS[@]}") + ;; + err) + "$@" 2>&1 >"$other" | tee "$file" >/dev/null + st=("${PIPESTATUS[@]}") + ;; + *) + echo "HARNESS ERROR: keep_stream $mode" >&2 + exit 2 + ;; + esac + ks_rc=${st[0]} + if [ "${st[1]}" -ne 0 ]; then + echo "HARNESS ERROR: the raw output $file could not be written (tee status ${st[1]}, command status ${st[0]})" >&2 + exit 2 + fi +} + +# l15c_result MS OK RC: the result line of L-15c's launch TERMed at MS ms, +# from its verdict OK (yes or no) and its status RC. The verdict's status +# is taken before the name is built: a command substitution in the name +# sets $? itself (up to 5bcdd1b the name held one, so every moment but +# 300 ms was reported failed and 300 ms passed, whatever the launch did). +# T-44. +l15c_result() { + local ok name + [ "$2" = yes ] + ok=$? + name="L-15c TERM at $1 ms: an allowed outcome" + [ "$1" != 300 ] || name="$name; no profile of these launches is left" + result "$name" "$ok" "(rc $3)" +} + +# t07_verdict STATUS FILE: T-07's verdict on its launch: the launch ended 0 +# (STATUS, the launch's own: ks_rc), the filter of its kept stdout ended 0, +# and that stdout without the workflow command lines is exactly "refused +# reached" (left in t07). Up to 5bcdd1b T-07 took grep's status for the +# launch's, so a launch that printed the text and then ended 125 passed. +# T-47. +t07_verdict() { + local filtered + t07=$(grep -v '^::' "$2") + filtered=$? + [ "$1" -eq 0 ] && [ "$filtered" -eq 0 ] && [ "$t07" = "refused reached" ] +} + +# l07_py: L-07's caller. It runs the launcher ($2) with /usr/bin/true and, +# as its stdout, one end of a socket pair or the slave of a new +# pseudo-terminal ($1), and no descriptor above 2. The pty's master stays +# open in this process, which drains it until the launcher has ended and +# then exits with its status (128 + the signal if it was killed): a slave +# whose master is closed is hung up, and isatty() does not report a hung-up +# terminal on Linux. (Up to 5bcdd1b the master was closed by the exec, so +# the launcher saw no terminal and refused only when its first write +# failed, after it had made its resources.) Bounded: after 150 s the +# launcher is killed and the status is 1. T-45. +l07_py=' +import errno, os, pty, select, signal, socket, sys, time +kind, launcher = sys.argv[1], sys.argv[2] +fddir = "/proc/self/fd" if os.path.isdir("/proc/self/fd") else "/dev/fd" +def run(out): + os.dup2(out, 1, inheritable=True) + for fd in sorted(int(n) for n in os.listdir(fddir)): + if fd > 2: + try: + os.close(fd) + except OSError as e: + if e.errno != errno.EBADF: + os._exit(2) + os.execv(launcher, [launcher, "/usr/bin/true"]) +if kind == "socketpair": + a, b = socket.socketpair() + run(a.fileno()) +master, slave = pty.openpty() +pid = os.fork() +if pid == 0: + run(slave) +os.close(slave) +end = time.monotonic() + 150 +while True: + done, status = os.waitpid(pid, os.WNOHANG) + if done: + sys.exit(os.WEXITSTATUS(status) if os.WIFEXITED(status) else 128 + os.WTERMSIG(status)) + if time.monotonic() > end: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + sys.exit(1) + if select.select([master], [], [], 0.1)[0]: + try: + os.read(master, 65536) + except OSError: + pass +' + +# l18_lines FILE: the number of complete lines of FILE that are exactly +# 65535 times "e" and their newline (L-18's 64 KiB of stderr), compared +# byte for byte, each line running from the start of FILE or a newline to +# the next newline; "unterminated" if FILE does not end with a newline (its +# last line is incomplete: no count is given); "-" if FILE cannot be read +# (missing, a directory). No pattern: GNU grep refuses the repetition bound +# of '^e\{65535\}$' ("Regular expression too big"), so the earlier count +# was empty on Linux; it also counted a last line without its newline. +# T-46. +l18_lines() { + python3 -I -c ' +import sys +*lines, rest = open(sys.argv[1], "rb").read().split(b"\n") +print("unterminated" if rest else sum(1 for line in lines if line == b"e" * 65535)) +' "$1" 2>/dev/null || echo - +} + +# --- the helpers ------------------------------------------------------------- +# Tracked files of this suite (scripts/ci/tests/no-docker/), copied to +# $helpers and checked against the sha256 below, which is part of this +# reviewed source: a copy that differs in any way is never run. Changing +# a helper needs a new review and a new hash here. +# canary.py : the canary engine, a Unix socket that answers +# like Docker and records every request; +# drive.sh : +# starts the canary (logging to /canary.log), records that +# the harness was started (/drive-started), runs it with +# DOCKER_HOST pointing at the canary and stops the canary again; +# payload: the mutant's payload (below). +helper_sums="canary.py=35dae9af68f8ec8a8a8668a1468fb192ed14f35461d893e3ab9525db11d69ce5 drive.sh=bb13bfa8771d611cc6be3bacf709b769b0c7d0868f0d2821b040107019a7d283 payload=7fd727d79f1eb2e8e3a6f52e25da8f0fe35851b79d262384f6bbc5f5c946ed25" +cp "$here/no-docker/canary.py" "$here/no-docker/drive.sh" "$here/no-docker/payload" "$helpers/" || exit 2 + +# A Docker configuration whose context "canary" points at a canary socket. +ctx=$(printf 'canary' | { if command -v sha256sum >/dev/null 2>&1; then sha256sum; else shasum -a 256; fi; } | cut -c1-64) +ctxcfg() { # dir canary + mkdir -p "$1/contexts/meta/$ctx" || return 1 + printf '{"Name":"canary","Metadata":{},"Endpoints":{"docker":{"Host":"unix://%s","SkipTLSVerify":false}}}\n' \ + "$2" >"$1/contexts/meta/$ctx/meta.json" +} + +# --- the mutant --------------------------------------------------------------- +# The payload is inserted after the first line of bin/devenv, so it runs as +# soon as the file is sourced or executed. Each case appends +# " rc= out=" to the log; the mutant then exits 97. +# It is exactly the reviewed text: on macOS the cases T-03 and T-04 run +# without the outer guard, so their safety rests on it only reading. + +# verify_helpers : the helpers and the payload in are regular +# files with the reviewed sha256 of helper_sums. Called right after they +# are copied, immediately before every run of drive.sh and before every +# mutant is made. +verify_helpers() { + local f want got + for f in canary.py drive.sh payload; do + want=${helper_sums#*"$f="} + want=${want%% *} + if [ ! -f "$1/$f" ] || [ -L "$1/$f" ]; then + echo "HARNESS ERROR: $1/$f is not a regular file; refusing to run" >&2 + return 1 + fi + got=$(sha "$1/$f") + if [ "$got" != "$want" ]; then + echo "HARNESS ERROR: $1/$f is not the reviewed one ($got); refusing to run" >&2 + return 1 + fi + done +} +verify_helpers "$helpers" || exit 2 + +# bare : drive.sh outside any +# isolation, only after its helpers verified. +bare() { + local dir=$1 + shift + verify_helpers "$dir" || return 2 + "$BASH" "$dir/drive.sh" "$dir/canary.py" "$@" +} + +# The paths substituted into the mutant and the profiles are plain. +for p in "$work" "$ro" "$vt" "$home_marker" "$tmp"; do + case $p in *[!A-Za-z0-9/._-]*) echo "HARNESS ERROR: unexpected character in $p; refusing to run" >&2; exit 2 ;; esac +done + +# mutant : print bin/devenv with the payload +# after line 1. +mutant() { + verify_helpers "$helpers" || return 1 + sed -n '1p' "$1" + sed -e "s|@LOG@|'$2'|" -e "s|@CANARY@|'$3'|" -e "s|@CFG@|'$4'|" -e "s|@MARKER@|'$home_marker'|" "$helpers/payload" + sed '1d' "$1" +} + +# tree : a copy of the files the harness needs, with +# as tests/devenv/unit.sh. +tree() { + mkdir -p "$1/bin" "$1/tests/devenv" "$1/scripts/ci" + cp "$root/bin/devenv" "$1/bin/devenv" + cp "$2" "$1/tests/devenv/unit.sh" + cp "$root/tests/devenv/tally.sh" "$root/tests/devenv/port-holder.py" "$1/tests/devenv/" + cp "$root/scripts/ci/no-docker.sh" "$root/scripts/ci/no-docker-probe.py" "$1/scripts/ci/" + chmod +x "$1/bin/devenv" "$1/tests/devenv/unit.sh" "$1/scripts/ci/no-docker.sh" +} + +# run_case : run the harness +# against the mutant; sets RC, LOG, OUT, STARTED (the harness was started) +# and REQ (requests the canary engine received). +# macOS: the canary and drive.sh's records are in , outside the +# isolation; the mutant's log is in the run directory W the harness's +# no-docker.sh gets (NO_DOCKER_WORK), the only place it may write. +# Linux: the guard is `no-docker.sh --work G`; the canary, the records +# and the log are in G, the tree and helpers are visible read-only. +run_case() { + local name=$1 harness=$2 how=$3 mode=$4 t=$ro/$1 g="" w="" data canary cmd + tree "$t" "$harness" + if [ "$os" = Linux ]; then + g=$(new_work) || exit 2 + data=$g/d + LOG=$data/mutant.log + else + data=$work/$name + mkdir "$data" || exit 2 + w=$(new_work) || exit 2 + LOG=$w/mutant.log + fi + canary=$data/c + OUT=$work/$name.out + ctxcfg "$t/ctxcfg" "$canary" || exit 2 + mutant "$root/bin/devenv" "$LOG" "$canary" "$t/ctxcfg" >"$t/mutant" || exit 2 + [ "$how" = inplace ] && cp "$t/mutant" "$t/bin/devenv" + cmd=(/usr/bin/env) + [ -z "$w" ] || cmd+=("NO_DOCKER_WORK=$w") + [ "$how" = inplace ] || cmd+=("DEVENV=$t/mutant") + cmd+=("$BASH" "$t/tests/devenv/unit.sh") + rm -f "$home_marker" + case $os-$mode in + Darwin-bare) bare "$helpers" "$data" "$canary" "${cmd[@]}" ;; + Darwin-guard) + # The write policy of no-docker.sh: only the case's directories and + # the /dev nodes of its profile; plus its other denies and the ports + # listening now. Only the canary socket is reachable. The guard's + # sandbox carries markers like no-docker.sh's, and no-docker-probe.py + # supervises it the same way: whatever the harness leaves running is + # ended when drive.sh returns, and the case fails if that cannot be + # shown (125). + verify_helpers "$helpers" || exit 2 + held="" + for port in $(python3 -I "$root/scripts/ci/no-docker-probe.py" --listening) ; do + case $port in *[!0-9]* | "") echo "HARNESS ERROR: unexpected port $port" >&2; exit 2 ;; esac + held="$held (remote ip \"localhost:$port\")" + done + gmark=$(python3 -I "$root/scripts/ci/no-docker-probe.py" --markers "$ndtmp" "$$") || exit 2 + case $gmark in "$ndtmp"/no-docker-mark.?*) ;; *) echo "HARNESS ERROR: unexpected marker directory $gmark" >&2; exit 2 ;; esac + python3 -I "$root/scripts/ci/no-docker-probe.py" --supervise "$gmark" "$data/drive-started" - -- /usr/bin/sandbox-exec -p "(version 1) +(allow default) +(deny network-outbound (remote unix-socket)) +(allow network-outbound (remote unix-socket (path-literal \"$canary\"))) +(deny network-outbound (remote ip)) +(allow network-outbound (remote ip \"localhost:*\")) +(deny network-outbound (remote ip \"localhost:22\") (remote ip \"localhost:2375\") (remote ip \"localhost:2376\")$held) +(deny file-write*) +(allow file-write* (subpath \"$data\") (subpath \"$w\") (literal \"/dev/null\") (literal \"/dev/zero\") (literal \"/dev/tty\") (literal \"/dev/dtracehelper\") (regex #\"^/dev/fd/[0-9]+\$\")) +(deny file-read-data (literal \"$gmark/a\")) +(deny file-read* (regex #\"^/dev/ttys[0-9]+\$\")) +(deny user-preference-write) +(deny file-ioctl (ioctl-command #x80017472)) +(deny system-debug) +(deny mach-task-name mach-task-read mach-task-inspect mach-priv-task-port) +(allow mach-task-name mach-task-read mach-task-inspect mach-priv-task-port (target same-sandbox)) +(deny signal) +(allow signal (target same-sandbox)) +(deny process-exec (literal \"/usr/bin/open\") (literal \"/usr/bin/osascript\") (literal \"/bin/launchctl\") (literal \"/usr/bin/ssh\")) +(deny job-creation) +(deny lsopen) +(deny appleevent-send) +(deny mach-lookup (global-name \"com.apple.coreservices.launchservicesd\") (global-name \"com.apple.coreservices.appleevents\") (global-name-regex #\"^com\\.apple\\.lsd\\.\") (global-name \"com.apple.xpc.smd\") (global-name-regex #\"^com\\.apple\\.backgroundtaskmanagement\") (global-name-regex #\"^com\\.apple\\.coreservices\\.sharedfilelistd\") (global-name-regex #\"^com\\.docker\\.\") (global-name-regex #\"^com\\.electron\\.dockerdesktop\\.\"))" \ + "$BASH" "$helpers/drive.sh" "$helpers/canary.py" "$data" "$canary" "${cmd[@]}" + ;; + *) + verify_helpers "$helpers" || exit 2 + count_launch + "$nd" --work "$g" "$BASH" "$helpers/drive.sh" "$helpers/canary.py" "$data" "$canary" "${cmd[@]}" + ;; + esac >"$OUT" 2>&1 + RC=$? + STARTED=1 + [ -e "$data/drive-started" ] || STARTED=0 + REQ=$(count_in . "$data/canary.log") +} + +# The harness without the containment: the lines that re-execute it under +# no-docker.sh removed. +sed '/^# The suite sources and runs bin\/devenv/,/^ exec .*scripts\/ci\/no-docker.sh" /d' \ + "$root/tests/devenv/unit.sh" >"$ro/unit-uncontained.sh" +if cmp -s "$ro/unit-uncontained.sh" "$root/tests/devenv/unit.sh" || grep -q no-docker "$ro/unit-uncontained.sh" || + ! grep -q '^\. "\$devenv_script"$' "$ro/unit-uncontained.sh"; then + echo "HARNESS ERROR: could not remove the containment from tests/devenv/unit.sh" >&2 + exit 2 +fi +harness=${HARNESS:-"$root/tests/devenv/unit.sh"} +# Another harness is unreviewed code: run every case behind the guard. +t0304=bare +[ -z "${HARNESS-}" ] || t0304=guard + +# read_only : lint (in addition to the hash): every docker command is +# `version` (the xargs case gets it on stdin), and no word of a command that +# changes anything appears (Docker subcommands, HTTP methods other than GET). +# A static check cannot be complete; the hash above is what is relied on. +read_only() { + local words + ! grep -qE '(^|[[:space:]"(=;|&])(rm|rmi|prune|kill|stop|restart|pause|create|run|start|push|pull|build|buildx|builder|load|tag|commit|cp|attach|update|rename|login|logout|volume|network|system|compose|swarm|service|stack|secret|config|plugin|image|container|DELETE|POST|PUT|PATCH|-XDELETE|-XPOST|-XPUT|-d|--data|--json|-F|-T)([[:space:]"'"'"');|&]|$)' "$1" || return 1 + words=$(grep -oE '"?(docker|\$qa_dk|\$qa_p)"?( -H "unix://[^"]*"| -H unix:///var/run/docker.sock| --context canary)? [a-z]+' "$1" | + awk '{print $NF}' | sort -u | tr '\n' ' ') + [ "$words" = "version " ] +} +read_only "$helpers/payload" || { echo "HARNESS ERROR: the mutant payload fails the read-only lint; refusing to run" >&2; exit 2; } + +# --- T-00 the read-only check itself (payload copies are only read here) +t00_bad=0 +for extra in 'docker image ls' '"docker" rmi x' 'docker system df' 'qa_x=$(docker volume ls)' \ + 'curl -X DELETE --unix-socket "$qa_canary" http://d/images/x' "python3 -c 's.sendall(b\"POST /x\")'" \ + 'docker exec c true' 'docker --context canary info' 'curl -d x --unix-socket "$qa_canary" http://d/x'; do + { cat "$helpers/payload"; printf '%s\n' "$extra"; } >"$work/payload-t00" + if read_only "$work/payload-t00"; then + t00_bad=$((t00_bad + 1)) + printf 'INFO T-00 accepted: %s\n' "$extra" + fi +done +[ "$t00_bad" -eq 0 ] +result "T-00 the read-only lint rejects a payload with a non-read-only command (9 variants)" $? + +# --- T-01/T-02 no-docker.sh itself +"$nd" --check >"$work/t01.out" 2>&1 +r=$? +[ "$r" -ne 0 ] +result "T-01 --check fails outside the isolation" $? "(rc $r)" +# Refused for the right reason: the probe's mechanism check failed, and +# its part that writes and connects never ran. +keep_stream both "$work/t02.out" env NO_DOCKER_ACTIVE=1 "$nd" touch "$work/t02" +r=$ks_rc +t02=$(cat "$work/t02.out") +t02_after=$(printf '%s\n' "$t02" | grep -cE 'container engine answers|accepts a connection|writes to .* fail') +[ "$r" -eq 125 ] && [ ! -e "$work/t02" ] && + printf '%s\n' "$t02" | grep -q '^no-docker: refused: inside no-docker.sh, but the isolation check fails' && + printf '%s\n' "$t02" | grep -q '^no-docker: FAIL ' && [ "$t02_after" -eq 0 ] +result "T-02 claimed but unproven isolation (NO_DOCKER_ACTIVE outside it): refused by the mechanism check, command not run" $? \ + "(rc $r, $(printf '%s\n' "$t02" | grep -c '^no-docker: FAIL ') failed checks, $t02_after lines of the write and connect part)" + +if [ "$os" = Darwin ]; then + # --- T-06 plain commands (no mutant): the write limits. A bash 3.2 + # here-document from a working directory outside the run's directory + # fails: bash keeps it in /tmp (it cannot use /var/tmp inside), which the + # sandbox does not allow, so code run inside uses none. + # On dfb6ee4: /private/tmp/sh-thd* was writable, so the here-document + # worked (rc 0 and "here-document" in t06). + w6=$(new_work) || exit 2 + "$nd" --work "$w6" /bin/bash -c 'touch "$1" "$2" "$3" 2>/dev/null; cd / && cat >"$4" <"$work/t06.out" 2>&1 + r=$? + [ "$r" -eq 1 ] && [ ! -s "$w6/t06" ] && grep -q 'cannot create temp file for here.document' "$work/t06.out" && + [ ! -e "$home_marker" ] && [ ! -e "$outside_marker" ] && [ ! -e "$tmp_marker" ] + result "T-06 writes only in the run's directory: not under \$HOME, next to \$TMPDIR or in /tmp (a bash 3.2 here-document fails)" $? "(rc $r)" + rm -f "$home_marker" "$outside_marker" "$tmp_marker" + + # T-08: the held-port list must be complete. A netstat without its column + # header, or with a LISTEN row that cannot be parsed, refuses; one with + # the header and no LISTEN row is a provably empty list. + mkdir -p "$work/t08" + t08() { # name netstat-output expected-rc + local w + printf '#!/bin/sh\nprintf "%%b" %s\n' "'$2'" >"$work/t08/netstat" + chmod +x "$work/t08/netstat" + w=$(new_work) || exit 2 + PATH=$work/t08:$PATH "$nd" --work "$w" /usr/bin/touch "$w/ran" >/dev/null 2>&1 + r=$? + if [ "$3" -eq 0 ]; then + [ "$r" -eq 0 ] && [ -e "$w/ran" ] + else + [ "$r" -eq 125 ] && [ ! -e "$w/ran" ] + fi + result "T-08 $1" $? "(rc $r)" + } + hdr='Active Internet connections (including servers)\nProto Recv-Q Send-Q Local Address Foreign Address (state)\n' + t08 "netstat without its column header: refused, command not run" 'tcp4 0 0 127.0.0.1.1 *.* LISTEN\n' 125 + t08 "an unparsable LISTEN row: refused, command not run" "${hdr}tcp4 0 0 garbage *.* LISTEN\n" 125 + t08 "header and no LISTEN row: an empty list is accepted" "${hdr}tcp4 0 0 127.0.0.1.1 127.0.0.1.2 ESTABLISHED\n" 0 + + # T-09: $TMPDIR must not be /tmp. + TMPDIR=/tmp "$nd" /usr/bin/touch "$work/t09" >/dev/null 2>&1 + r=$? + [ "$r" -eq 125 ] && [ ! -e "$work/t09" ] + result "T-09 TMPDIR=/tmp: refused, command not run" $? "(rc $r)" + + # T-10: no preference written through cfprefsd, no keystroke injected + # into the terminal (TIOCSTI), run in a pseudo-terminal of its own. + t10_domain=com.flowcrm.no-docker-test-$nonce + "$nd" /usr/bin/defaults write "$t10_domain" k v >/dev/null 2>&1 + t10_pref=$(defaults read "$t10_domain" k 2>/dev/null) + defaults delete "$t10_domain" >/dev/null 2>&1 + rm -f "$home/Library/Preferences/$t10_domain.plist" + [ -z "$t10_pref" ] + result "T-10 defaults write inside the isolation changes no preference" $? + t10=$(script -q /dev/null "$nd" python3 -I -c ' +import fcntl, os +fd = os.open("/dev/tty", os.O_RDWR) +try: + fcntl.ioctl(fd, 0x80017472, b" ") + print("t10-injected") +except OSError as e: + print("t10-denied", e.errno) +' /dev/null | tr -d '\r') + case $t10 in *"t10-denied 1"*) r=0 ;; *) r=1 ;; esac + result "T-10 TIOCSTI on the controlling terminal is denied" $r "($(printf '%s' "$t10" | grep t10-))" + + # --- T-11 writes in $TMPDIR outside the run's directory are denied; in it + # they work, and the command's $TMPDIR is inside it. + # On 38b5ebd: the profile allowed the whole $TMPDIR, so the marker in + # $TMPDIR is written (and --work is not an option there: rc 127). + w11=$(new_work) || exit 2 + "$nd" --work "$w11" /bin/sh -c 'touch "$1" 2>/dev/null; touch "$2" && touch "$TMPDIR/t11"' _ "$t11_marker" "$w11/inside" >/dev/null 2>&1 + r=$? + [ "$r" -eq 0 ] && [ ! -e "$t11_marker" ] && [ -e "$w11/inside" ] && [ -e "$w11/tmp/t11" ] + result "T-11 --work: writes in the run's directory and its \$TMPDIR work, in \$TMPDIR outside it are denied" $? "(rc $r)" + rm -f "$t11_marker" + t11=$("$nd" /bin/sh -c 'touch "$1" 2>/dev/null; touch "$TMPDIR/t11" && printf "%s" "$TMPDIR"' _ "$t11_marker" 2>/dev/null) + r=$? + case $t11 in "$tmp"/no-docker.*/tmp) t11ok=0 ;; *) t11ok=1 ;; esac + [ "$r" -eq 0 ] && [ "$t11ok" -eq 0 ] && [ ! -e "$t11_marker" ] + result "T-11 without --work: \$TMPDIR is in a fresh directory of the run; \$TMPDIR itself is not writable" $? "(rc $r, TMPDIR $t11)" + rm -f "$t11_marker" + + # --- T-12 $TMPDIR allowlist: only the per-user /private/var/folders/../T. + # On 38b5ebd: a denylist accepted a user-owned directory that is not that + # one, and /private/var/tmp; the command ran there (rc 0, marker written). + mkdir -m 700 "$work/t12" "$work/t12/T" + t12v=$(mktemp -d /private/var/tmp/no-docker-tests-t12.XXXXXX) || exit 2 + t12() { # name tmpdir expected-rc + local m=$2/.t12-ran-$nonce + if [ "$3" -eq 0 ]; then + if [ -n "$2" ]; then env TMPDIR="$2" "$nd" /usr/bin/true; else env -u TMPDIR "$nd" /usr/bin/true; fi >/dev/null 2>&1 + r=$? + [ "$r" -eq 0 ] + else + env TMPDIR="$2" "$nd" /usr/bin/touch "$m" >/dev/null 2>&1 + r=$? + [ "$r" -eq 125 ] && [ ! -e "$m" ] + fi + result "T-12 $1" $? "(rc $r)" + rm -f "$m" + } + t12 "a user-owned directory named T, not the per-user one: refused" "$work/t12/T" 125 + t12 "/private/var/tmp/: refused" "$t12v" 125 + t12 "/private/var/tmp: refused" /private/var/tmp 125 + t12 "/: refused" / 125 + t12 "the per-user directory: accepted" "$tmp" 0 + t12 "TMPDIR unset (getconf DARWIN_USER_TEMP_DIR): accepted" "" 0 + rmdir "$t12v" 2>/dev/null + + # --- T-13 signals: a canary `sleep` started here, outside, cannot be + # signalled from inside (kill -0 gives EPERM) and is alive afterwards; the + # command can signal its own child and its own process group. + # On 38b5ebd: the profile had no signal rule, so kill -0 succeeded. + sleep 120 & + t13pid=$! + t13=$("$nd" python3 -I -c ' +import os, signal, subprocess, sys +try: + os.kill(int(sys.argv[1]), 0) + print("outside-reached") +except PermissionError: + print("outside-eperm") +except OSError as e: + print("outside-error", e.errno) +c = subprocess.Popen(["/bin/sleep", "30"]) +os.kill(c.pid, signal.SIGTERM) +print("own-child", c.wait()) +g = subprocess.Popen(["/bin/sleep", "30"], preexec_fn=os.setpgrp) +os.killpg(g.pid, signal.SIGTERM) +print("own-group", g.wait()) +' "$t13pid" 2>/dev/null | tr '\n' ' ') + r=$? + kill -0 "$t13pid" 2>/dev/null + alive=$? + kill -TERM "$t13pid" 2>/dev/null + wait "$t13pid" 2>/dev/null + [ "$t13" = "outside-eperm own-child -15 own-group -15 " ] && [ "$alive" -eq 0 ] + result "T-13 no signal to a process outside (EPERM, canary alive); own child and process group can be signalled" $? "($t13, canary alive rc $alive)" + + # --- T-14 integrity of what this suite runs outside any isolation. + # On 38b5ebd: the helpers were in $TMPDIR, which the sandbox allowed to + # write, so the append below succeeded. + cp "$helpers/drive.sh" "$ro/t14-target" + t14_before=$(sha "$ro/t14-target") + t14=$("$nd" /bin/sh -c 'printf "# changed\n" >>"$1" 2>/dev/null; echo "append-rc=$?"' _ "$ro/t14-target" 2>/dev/null) + [ "$t14" = "append-rc=1" ] && [ "$(sha "$ro/t14-target")" = "$t14_before" ] + result "T-14 the suite's helpers cannot be changed from inside the isolation" $? "($t14)" + # The check before a bare run, on a tampered copy: drive.sh is not run. + mkdir "$work/t14-copy" + cp "$helpers/canary.py" "$helpers/drive.sh" "$helpers/payload" "$work/t14-copy/" + printf '# tampered\n' >>"$work/t14-copy/drive.sh" + bare "$work/t14-copy" "$work/t14-data" "$work/t14-data/c" /usr/bin/touch "$work/t14-ran" >/dev/null 2>&1 + r=$? + verify_helpers "$helpers" 2>/dev/null + intact=$? + [ "$r" -eq 2 ] && [ ! -e "$work/t14-ran" ] && [ ! -e "$work/t14-data" ] && [ "$intact" -eq 0 ] + result "T-14 a changed helper is detected before a bare run, which is then not made" $? "(rc $r, intact helpers $intact)" + + # --- T-15 --work DIR: accepted for a fresh, empty, own 0700 directory + # under $TMPDIR or /private/var/tmp (writes work there and only there); + # refused otherwise. (New option: on 38b5ebd "--work" was run as the + # command, rc 127, so every case fails there.) + w15=$(new_work) || exit 2 + "$nd" --work "$w15" /bin/sh -c 'touch "$1" && { touch "$2" 2>/dev/null; true; }' _ "$w15/inside" "$work/t15-outside" >/dev/null 2>&1 + r=$? + [ "$r" -eq 0 ] && [ -e "$w15/inside" ] && [ ! -e "$work/t15-outside" ] + result "T-15 --work under \$TMPDIR: accepted, writable, nothing outside it" $? "(rc $r)" + w15v=$(mktemp -d /private/var/tmp/no-docker-tests-t15.XXXXXX) || exit 2 + "$nd" --work "$w15v" /usr/bin/touch "$w15v/inside" >/dev/null 2>&1 + r=$? + [ "$r" -eq 0 ] && [ -e "$w15v/inside" ] + result "T-15 --work under /private/var/tmp: accepted" $? "(rc $r)" + rm -rf "$w15v" + mkdir -m 700 "$work/t15" "$work/t15/full" "$work/t15/m755" "$work/t15/real" "$work/t15/real/w" + chmod 755 "$work/t15/m755" + : >"$work/t15/full/x" + ln -s "$work/t15/real" "$work/t15/link" + ln -s "$work/t15" "$work/t15/plink" + t15() { # name dir + local out + out=$("$nd" --work "$2" /bin/echo ran 2>/dev/null) + r=$? + [ "$r" -eq 125 ] && [ -z "$out" ] + result "T-15 --work $1: refused, command not run" $? "(rc $r)" + } + t15 "a symbolic link" "$work/t15/link" + t15 "with a symbolic link in its path" "$work/t15/plink/real/w" + t15 "a non-empty directory" "$work/t15/full" + t15 "mode 0755" "$work/t15/m755" + t15 "\$HOME" "$home" + t15 "/tmp" /tmp + t15 "/private/var/tmp itself" /private/var/tmp + t15 "a relative path" t15 + + # --- T-21 here-document files of bash 3.2 cannot be changed from inside. + # Outside the sandbox bash 3.2 keeps them in /private/var/tmp, inside it + # (and in other sandboxes that allow /tmp but not /var/tmp) in + # /private/tmp. Two canaries named like them, made here: none can be + # opened for writing, truncated, appended to, removed, renamed or + # replaced from inside, and no new one can be created. + # On dfb6ee4: the profile allowed writes to /private/tmp/sh-thd*, so every + # operation on that canary succeeded. + # Both directories are shared with other users: each canary gets a + # random name and is created exclusively, without following a link; it + # is read and removed only while it is still the file created here + # (same device and inode, a regular file of ours). + t21_file=' +import os, secrets, stat, sys +mode = sys.argv[1] +def same(path, dev, ino): + st = os.lstat(path) + return stat.S_ISREG(st.st_mode) and st.st_uid == os.getuid() and (st.st_dev, st.st_ino) == (int(dev), int(ino)) +if mode == "create": + path = os.path.join(sys.argv[2], "sh-thd-flowcrm-test-" + secrets.token_hex(12)) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + os.write(fd, b"original\n") + st = os.fstat(fd) + os.close(fd) + print(path, st.st_dev, st.st_ino) +elif mode == "read": + path, dev, ino = sys.argv[2:5] + try: + if not same(path, dev, ino): + print("replaced") + sys.exit(0) + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + print("same" if (os.fstat(fd).st_dev, os.fstat(fd).st_ino) == (int(dev), int(ino)) and os.read(fd, 64) == b"original\n" else "changed") + os.close(fd) + except FileNotFoundError: + print("missing") +elif mode == "remove": + path, dev, ino = sys.argv[2:5] + try: + if same(path, dev, ino): + os.unlink(path) + except FileNotFoundError: + pass +' + read -r t21a t21a_dev t21a_ino < <(python3 -I -c "$t21_file" create /private/tmp) || exit 2 + read -r t21b t21b_dev t21b_ino < <(python3 -I -c "$t21_file" create /private/var/tmp) || exit 2 + case "$t21a $t21b" in /private/tmp/sh-thd-flowcrm-test-*" "/private/var/tmp/sh-thd-flowcrm-test-*) ;; *) echo "HARNESS ERROR: T-21 canaries not created" >&2; exit 2 ;; esac + w21=$(new_work) || exit 2 + t21=$("$nd" --work "$w21" python3 -I -c ' +import errno, os, sys +w = sys.argv[1] +for path in sys.argv[2:]: + open(os.path.join(w, "x"), "w").close() + tries = { + "write": lambda: os.open(path, os.O_WRONLY), + "read-write": lambda: os.open(path, os.O_RDWR), + "append": lambda: os.open(path, os.O_WRONLY | os.O_APPEND), + "truncate": lambda: os.truncate(path, 0), + "unlink": lambda: os.unlink(path), + "rename-away": lambda: os.rename(path, path + ".moved"), + "replace": lambda: os.rename(os.path.join(w, "x"), path), + "create": lambda: os.open(path + ".new", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), + "chmod": lambda: os.chmod(path, 0o666), + } + for name, f in tries.items(): + try: + r = f() + if isinstance(r, int): + os.close(r) + print("t21 bad ", path, name, "allowed") + except OSError as e: + print("t21 " + ("ok " if e.errno in (errno.EPERM, errno.EACCES) else "bad ") , path, name, + errno.errorcode.get(e.errno, e.errno)) +' "$w21" "$t21a" "$t21b" 2>/dev/null) + r=$? + t21_left="" + t21s=$(python3 -I -c "$t21_file" read "$t21a" "$t21a_dev" "$t21a_ino") + [ "$t21s" = same ] || t21_left="$t21_left $t21s:$t21a" + t21s=$(python3 -I -c "$t21_file" read "$t21b" "$t21b_dev" "$t21b_ino") + [ "$t21s" = same ] || t21_left="$t21_left $t21s:$t21b" + for f in "$t21a" "$t21b"; do + for x in "$f.moved" "$f.new"; do [ ! -e "$x" ] && [ ! -L "$x" ] || t21_left="$t21_left $x"; done + done + printf '%s\n' "$t21" | grep '^t21 bad' | sed 's/^/INFO /' + [ "$r" -eq 0 ] && [ "$(printf '%s\n' "$t21" | grep -c '^t21 ok')" -eq 18 ] && [ -z "$t21_left" ] + result "T-21 bash 3.2 here-document files (/private/tmp and /private/var/tmp sh-thd-*) cannot be written, replaced or created from inside" $? \ + "(rc $r, $(printf '%s\n' "$t21" | grep -c '^t21 ok') of 18 denied, changed or left:${t21_left:- none})" + python3 -I -c "$t21_file" remove "$t21a" "$t21a_dev" "$t21a_ino" + python3 -I -c "$t21_file" remove "$t21b" "$t21b_dev" "$t21b_ino" + + # --- T-22 a file outside the run's directory cannot be changed through a + # hard link, a symbolic link, a rename over it or an exchange (all on + # the same volume as the run's directory). + # (A-M3: measured on 38b5ebd, no regression case until now.) + t22f=$work/t22-target + printf 'original\n' >"$t22f" || exit 2 + w22=$(new_work) || exit 2 + t22=$("$nd" --work "$w22" python3 -I -c ' +import ctypes, errno, os, sys +w, target = sys.argv[1], sys.argv[2] +libc = ctypes.CDLL(None, use_errno=True) +def say(ok, text): + print(("t22 ok " if ok else "t22 bad ") + text) +try: + os.link(target, os.path.join(w, "hard")) + try: + with open(os.path.join(w, "hard"), "a") as f: + f.write("changed\n") + say(False, "hard link made and written through") + except OSError as e: + say(True, f"hard link made, but not writable ({errno.errorcode.get(e.errno)})") +except OSError as e: + say(True, f"hard link refused ({errno.errorcode.get(e.errno)})") +os.symlink(target, os.path.join(w, "soft")) +try: + with open(os.path.join(w, "soft"), "a") as f: + f.write("changed\n") + say(False, "written through a symbolic link") +except OSError as e: + say(True, f"symbolic link not writable ({errno.errorcode.get(e.errno)})") +mine = os.path.join(w, "mine") +with open(mine, "w") as f: + f.write("replaced\n") +try: + os.rename(mine, target) + say(False, "renamed over the target") +except OSError as e: + say(True, f"rename over the target refused ({errno.errorcode.get(e.errno)})") +RENAME_SWAP = 0x2 +r = libc.renamex_np(os.fsencode(mine), os.fsencode(target), RENAME_SWAP) +say(r != 0, f"exchange with the target refused ({errno.errorcode.get(ctypes.get_errno(), r)})") +r = libc.exchangedata(os.fsencode(mine), os.fsencode(target), 0) +say(r != 0, f"exchangedata with the target refused ({errno.errorcode.get(ctypes.get_errno(), r)})") +' "$w22" "$t22f" 2>/dev/null) + r=$? + printf '%s\n' "$t22" | grep '^t22 ' | sed 's/^/INFO /' + [ "$r" -eq 0 ] && [ "$(printf '%s\n' "$t22" | grep -c '^t22 ok')" -eq 5 ] && [ "$(cat "$t22f")" = original ] + result "T-22 a file outside the run's directory cannot be changed through links, a rename or an exchange" $? \ + "(rc $r, target $(cat "$t22f" 2>/dev/null | head -n 1))" + + # --- T-24 to T-27 no process of the command outlives it. The command is a + # Python "leaver" that starts four processes meant to survive it: a plain + # child, a setsid daemon after a double fork, a posix_spawn child with + # POSIX_SPAWN_SETSID and a child in a process group of its own. Each + # appends its pid to /pids once it runs and then sleeps 60 s; + # every one carries the case's marker in its arguments, which is how the + # checks below (and their clean-up) recognise them. The leaver waits until + # all four are recorded, then ends as the case says. The pids are read + # back as data and a pid counts as a survivor only while its arguments + # still carry the marker. + # On dfb6ee4: nothing ended them, so all four survived each case. + t24_leaver=' +import ctypes, os, signal, sys, time +w, tag, how = sys.argv[1], sys.argv[2], sys.argv[3] +SLEEP = [sys.executable, "-I", "-B", "-c", "import os, sys, time; open(sys.argv[1], \"a\").write(f\"{os.getpid()}\\n\"); time.sleep(60)", w + "/pids", tag] +if os.fork() == 0: + os.execv(SLEEP[0], SLEEP) +if os.fork() == 0: + os.setsid() + if os.fork() == 0: + os.execv(SLEEP[0], SLEEP) + os._exit(0) +libc = ctypes.CDLL(None, use_errno=True) +attr = ctypes.c_void_p() +libc.posix_spawnattr_init(ctypes.byref(attr)) +libc.posix_spawnattr_setflags(ctypes.byref(attr), ctypes.c_short(0x0400)) # POSIX_SPAWN_SETSID +argv = (ctypes.c_char_p * (len(SLEEP) + 1))(*[a.encode() for a in SLEEP], None) +pid = ctypes.c_int() +libc.posix_spawn(ctypes.byref(pid), SLEEP[0].encode(), None, ctypes.byref(attr), argv, None) +if os.fork() == 0: + os.setpgid(0, 0) + os.execv(SLEEP[0], SLEEP) +for _ in range(100): + try: + if len(open(w + "/pids").read().split()) >= 4: + break + except OSError: + pass + time.sleep(0.05) +open(w + "/mark", "w").write(os.environ.get("NO_DOCKER_MARK", "")) +if how == "fail": + sys.exit(3) +if how == "kill-self": + os.kill(os.getpid(), signal.SIGKILL) +if how == "wait": + open(w + "/pids", "a").write(f"{os.getpid()}\n") + time.sleep(60) +if how == "linger": + open(w + "/started", "w").write(f"{time.time():.3f}\n") + time.sleep(3) +' + t24_left() { # run-dir tag: the recorded pids that still run as the case's processes + local p + regular "$1/pids" || return 0 + for p in $(tr -c '0-9\n' ' ' <"$1/pids"); do + ps -o args= -p "$p" 2>/dev/null | grep -q -- "$2" && printf ' %s' "$p" + done + } + t24_clean() { # run-dir tag: end only processes that still carry the case's marker + local p + for p in $(t24_left "$1" "$2"); do kill -KILL "$p" 2>/dev/null; done + } + t24() { # name how expected-rc + local w tag=t24-$2-$nonce left n + w=$(new_work) || exit 2 + "$nd" --work "$w" python3 -I -B -c "$t24_leaver" "$w" "$tag" "$2" >"$work/t24-$2.out" 2>&1 + r=$? + sleep 0.5 + left=$(t24_left "$w" "$tag") + n=$(count_in . "$w/pids") + t24_clean "$w" "$tag" + [ "$r" -eq "$3" ] && [ "$n" -eq 4 ] && [ -z "$left" ] && grep -q 'no-docker: ended [0-9]* processes' "$work/t24-$2.out" + result "T-24 $1: none of the 4 processes it left survives" $? "(rc $r, $n recorded, survivors:${left:- none})" + } + t24 "normal end" ok 0 + t24 "failed command" fail 3 + t24 "command killed by SIGKILL" kill-self 137 + + # T-25: SIGINT, SIGTERM or SIGHUP to no-docker.sh while the command runs + # ends the command and all it started at once. no-docker.sh is started + # with SIGINT at its default (an asynchronous command of this script + # would otherwise ignore it). + t25() { # signal expected-rc + local w tag=t25-$1-$nonce ndpid left n t0 took + w=$(new_work) || exit 2 + python3 -I -c 'import os, signal, sys +signal.signal(signal.SIGINT, signal.SIG_DFL) +os.execv(sys.argv[1], sys.argv[1:])' "$nd" --work "$w" python3 -I -B -c "$t24_leaver" "$w" "$tag" wait >"$work/t25-$1.out" 2>&1 & + ndpid=$! + for _ in $(seq 1 100); do [ "$(count_in . "$w/pids")" -ge 5 ] && break; sleep 0.1; done + n=$(count_in . "$w/pids") + t0=$(date +%s) + # Our own child, still running (its arguments name no-docker.sh). + ps -o args= -p "$ndpid" 2>/dev/null | grep -q -- no-docker && kill -"$1" "$ndpid" + wait "$ndpid" + r=$? + took=$(($(date +%s) - t0)) + sleep 0.5 + left=$(t24_left "$w" "$tag") + t24_clean "$w" "$tag" + [ "$n" -eq 5 ] && [ "$r" -eq "$2" ] && [ "$took" -le 10 ] && [ -z "$left" ] + result "T-25 SIG$1 to no-docker.sh ends the command and the 4 processes it left" $? \ + "(rc $r, $n recorded, ${took}s, survivors:${left:- none})" + } + t25 INT 130 + t25 TERM 143 + t25 HUP 129 + + # T-26: SIGKILL of no-docker.sh cannot be handled: what the command left + # keeps running, still sandboxed, and no new isolated run starts while it + # does (125, command not run). The case then ends them through the run's + # marker (no-docker-probe.py --end) and a new run starts again. + w=$(new_work) || exit 2 + tag=t26-$nonce + t26_before=$(cd "$ndtmp" && ls -d no-docker-mark.* 2>/dev/null) + python3 -I -c 'import os, signal, sys +signal.signal(signal.SIGINT, signal.SIG_DFL) +os.execv(sys.argv[1], sys.argv[1:])' "$nd" --work "$w" python3 -I -B -c "$t24_leaver" "$w" "$tag" wait >"$work/t26.out" 2>&1 & + ndpid=$! + for _ in $(seq 1 100); do [ "$(count_in . "$w/pids")" -ge 5 ] && break; sleep 0.1; done + ps -o args= -p "$ndpid" 2>/dev/null | grep -q -- no-docker && kill -KILL "$ndpid" + wait "$ndpid" 2>/dev/null + sleep 0.5 + t26_alive=$(t24_left "$w" "$tag") + w26=$(new_work) || exit 2 + "$nd" --work "$w26" /usr/bin/touch "$w26/ran" >"$work/t26b.out" 2>&1 + r=$? + t26_mark="" + regular "$w/mark" && t26_mark=$(cat "$w/mark") + # Data the sandboxed command wrote: accepted only as the name of a marker + # directory that did not exist before this case (one path component). + case ${t26_mark#"$ndtmp"/} in no-docker-mark.*[!A-Za-z0-9_]* | "$t26_mark") t26_mark="" ;; no-docker-mark.?*) ;; *) t26_mark="" ;; esac + case " +$t26_before +" in *" +${t26_mark#"$ndtmp"/} +"*) t26_mark="" ;; esac + [ -n "$t26_mark" ] && [ -d "$t26_mark" ] && [ ! -L "$t26_mark" ] || t26_mark="" + t26_end=1 + if [ -n "$t26_mark" ] && python3 -I "$root/scripts/ci/no-docker-probe.py" --end "$t26_mark" >/dev/null 2>&1; then + t26_end=0 + rm -rf "$t26_mark" + fi + t26_left=$(t24_left "$w" "$tag") + t24_clean "$w" "$tag" + w26c=$(new_work) || exit 2 + "$nd" --work "$w26c" /usr/bin/touch "$w26c/ran" >/dev/null 2>&1 + r2=$? + [ -n "$t26_alive" ] && [ "$r" -eq 125 ] && [ ! -e "$w26/ran" ] && grep -q 'earlier isolated run' "$work/t26b.out" && + [ "$t26_end" -eq 0 ] && [ -z "$t26_left" ] && [ "$r2" -eq 0 ] && [ -e "$w26c/ran" ] + result "T-26 after SIGKILL of no-docker.sh its leftovers keep every new run from starting until they are ended" $? \ + "(left after SIGKILL:${t26_alive:- none}; new run rc $r; ended through the marker $t26_end; afterwards rc $r2)" + + # T-27: if the processes of the isolation cannot be found or ended (here: + # no-docker-probe.py fails), nothing runs. + mkdir -p "$work/t27" + t27() { # mode + local w + printf '#!/bin/sh\ncase " $* " in *" %s "*) exit 1 ;; esac\nexec %s "$@"\n' "$1" "$(command -v python3)" >"$work/t27/python3" + chmod +x "$work/t27/python3" + w=$(new_work) || exit 2 + PATH=$work/t27:$PATH "$nd" --work "$w" /usr/bin/touch "$w/ran" >/dev/null 2>&1 + r=$? + [ "$r" -ne 0 ] && [ ! -e "$w/ran" ] + result "T-27 no-docker-probe.py $1 fails: the command is not run" $? "(rc $r)" + } + t27 --markers + t27 --orphans + t27 --supervise + + # --- T-30 to T-34 a run is reported done only when it is shown that none + # of its processes runs any more (B-1). no-docker-probe.py takes one copy + # of the process table at a time and needs a copy in which every process + # is shown not to be one of the run's, or was dead before that copy; + # otherwise the run fails with 125 and keeps its marker directory, and + # the next run refuses until that is shown. + # On e8ed7ed: one process table that happened to show no member ended + # the run as a success. + # + # T-30: a "hopper", each generation forking the next and ending, inside + # the sandbox: bounded (it ends by itself 18 s after it starts, or after + # 50000 generations) and recording a heartbeat in its run + # directory. The run must not be reported done while it hops; the next + # run must refuse while it hops; after it ended, a run starts again (and + # removes the old marker directory). + t30_hop=' +import os, sys, time +beat, deadline = sys.argv[1], float(sys.argv[2]) +g = 0 +while g < 50000 and time.time() < deadline: + g += 1 + if g % 100 == 0: + with open(beat, "a") as f: + f.write(f"{time.time():.3f} {g}\n") + if os.fork(): + os._exit(0) +' + t30_beats() { count_in . "$1"; } + w30=$(new_work) || exit 2 + t30_end=$(($(date +%s) + 18)) + t0=$(date +%s) + "$nd" --work "$w30" python3 -I -B -c "$t30_hop" "$w30/beat" "$t30_end" "t30-$nonce" >"$work/t30.out" 2>&1 + r=$? + took=$(($(date +%s) - t0)) + n0=$(t30_beats "$w30/beat") + sleep 1 + n1=$(t30_beats "$w30/beat") + t30_hopping=no + [ "$n1" -gt "$n0" ] && t30_hopping=yes + w30b=$(new_work) || exit 2 + "$nd" --work "$w30b" /usr/bin/touch "$w30b/ran" >"$work/t30b.out" 2>&1 + r2=$? + # Still hopping after that run returned? Then it must have refused. + n2=$(t30_beats "$w30/beat") + sleep 1 + t30_still=no + [ "$(t30_beats "$w30/beat")" -gt "$n2" ] && t30_still=yes + while [ "$(date +%s)" -le $((t30_end + 2)) ]; do sleep 1; done + w30c=$(new_work) || exit 2 + "$nd" --work "$w30c" /usr/bin/touch "$w30c/ran" >"$work/t30c.out" 2>&1 + r3=$? + # Not done while it hops: either the run failed (125, "could not be + # shown") or, if it ended anyway, the hopper had stopped. + { [ "$r" -eq 125 ] && grep -q 'could not be shown to have ended' "$work/t30.out"; } || + { [ "$r" -ne 125 ] && [ "$t30_hopping" = no ]; } + t30_ok1=$? + [ "$t30_ok1" -eq 0 ] && [ "$n0" -gt 0 ] && + { [ "$t30_still" = no ] || { [ "$r2" -eq 125 ] && [ ! -e "$w30b/ran" ]; }; } && + [ "$r3" -eq 0 ] && [ -e "$w30c/ran" ] + t30_ok=$? + echo "INFO T-30 run rc $r after ${took}s, hopping afterwards $t30_hopping; next run rc $r2, hopping after it $t30_still; after its end rc $r3" + [ "$t30_ok" -eq 0 ] + result "T-30 a process that keeps forking and ending is never reported as ended; new runs refuse while it runs, and start after it ended" $? \ + "(run rc $r after ${took}s, hopping afterwards $t30_hopping; next run rc $r2, hopping after it $t30_still; after its end rc $r3)" + + # T-31: two runs at the same time, each leaving a daemon: both are + # reported done, neither refuses the other, each ends its own daemon. + w31a=$(new_work) || exit 2 + w31b=$(new_work) || exit 2 + "$nd" --work "$w31a" python3 -I -B -c "$t24_leaver" "$w31a" "t31a-$nonce" linger >"$work/t31a.out" 2>&1 & + t31pa=$! + "$nd" --work "$w31b" python3 -I -B -c "$t24_leaver" "$w31b" "t31b-$nonce" linger >"$work/t31b.out" 2>&1 & + t31pb=$! + wait "$t31pa" + ra=$? + wait "$t31pb" + rb=$? + sleep 0.5 + la=$(t24_left "$w31a" "t31a-$nonce") + lb=$(t24_left "$w31b" "t31b-$nonce") + t24_clean "$w31a" "t31a-$nonce" + t24_clean "$w31b" "t31b-$nonce" + [ "$ra" -eq 0 ] && [ "$rb" -eq 0 ] && regular "$w31a/started" && regular "$w31b/started" && + [ "$(count_in . "$w31a/pids")" -eq 4 ] && [ "$(count_in . "$w31b/pids")" -eq 4 ] && [ -z "$la$lb" ] && + ! grep -q 'refused' "$work/t31a.out" "$work/t31b.out" + result "T-31 two runs at the same time each end their own processes and do not refuse each other" $? \ + "(rc $ra and $rb, survivors:${la}${lb:- none})" + + # T-32: unrelated processes that start and end while the run's + # processes are checked (here: this suite's own short-lived /usr/bin/true + # runs, outside) do not make the run fail: a process that ended before + # it could be checked is ended, not unknown, once a later copy of the + # table no longer has it. + w32=$(new_work) || exit 2 + ( i=0; while [ "$i" -lt 3000 ]; do /usr/bin/true; i=$((i + 1)); done ) & + t32churn=$! + "$nd" --work "$w32" python3 -I -B -c "$t24_leaver" "$w32" "t32-$nonce" ok >"$work/t32.out" 2>&1 + r=$? + # The churn must still have been running when the run ended. + t32_overlap=no + ps -o args= -p "$t32churn" 2>/dev/null | grep -q . && t32_overlap=yes + wait "$t32churn" + sleep 0.5 + left=$(t24_left "$w32" "t32-$nonce") + t24_clean "$w32" "t32-$nonce" + [ "$r" -eq 0 ] && [ "$t32_overlap" = yes ] && [ "$(count_in . "$w32/pids")" -eq 4 ] && [ -z "$left" ] + result "T-32 processes that start and end outside meanwhile do not make the ending fail" $? \ + "(rc $r, churn still running at the end $t32_overlap, survivors:${left:- none})" + + # T-33: if the ending cannot be shown (here: this case removes the run's + # marker file A while the command waits, then sends TERM), the run fails + # (125), the supervisor's own child does not survive, and the marker + # directory is kept. The case then ends the leftovers it recorded (by + # pid, while their arguments carry its tag), removes that directory and + # checks that a new run starts. + w33=$(new_work) || exit 2 + t33_before=$(cd "$ndtmp" && ls -d no-docker-mark.* 2>/dev/null) + python3 -I -c 'import os, signal, sys +signal.signal(signal.SIGINT, signal.SIG_DFL) +os.execv(sys.argv[1], sys.argv[1:])' "$nd" --work "$w33" python3 -I -B -c "$t24_leaver" "$w33" "t33-$nonce" wait >"$work/t33.out" 2>&1 & + ndpid=$! + for _ in $(seq 1 100); do [ "$(count_in . "$w33/pids")" -ge 5 ] && regular "$w33/mark" && break; sleep 0.1; done + t33_mark="" + regular "$w33/mark" && t33_mark=$(cat "$w33/mark") + case ${t33_mark#"$ndtmp"/} in no-docker-mark.*[!A-Za-z0-9_]* | "$t33_mark") t33_mark="" ;; no-docker-mark.?*) ;; *) t33_mark="" ;; esac + case " +$t33_before +" in *" +${t33_mark#"$ndtmp"/} +"*) t33_mark="" ;; esac + [ -n "$t33_mark" ] && [ -d "$t33_mark" ] && [ ! -L "$t33_mark" ] && [ -f "$t33_mark/a" ] && [ ! -L "$t33_mark/a" ] && + regular "$t33_mark/owner" && [ "$(cut -d' ' -f1 "$t33_mark/owner")" = "$ndpid" ] || t33_mark="" + t33_main=$(sed -n 5p "$w33/pids" 2>/dev/null | tr -cd '0-9') + r=none + if [ -n "$t33_mark" ] && ps -o args= -p "$ndpid" 2>/dev/null | grep -q -- no-docker; then + rm -f "$t33_mark/a" + kill -TERM "$ndpid" + wait "$ndpid" + r=$? + else + ps -o args= -p "$ndpid" 2>/dev/null | grep -q -- no-docker && kill -TERM "$ndpid" + wait "$ndpid" 2>/dev/null + fi + t33_child=gone + [ -n "$t33_main" ] && ps -o args= -p "$t33_main" 2>/dev/null | grep -q -- "t33-$nonce" && t33_child=alive + t33_kept=no + [ -n "$t33_mark" ] && [ -d "$t33_mark" ] && t33_kept=yes + t24_clean "$w33" "t33-$nonce" + [ -n "$t33_mark" ] && rm -rf "$t33_mark" + w33b=$(new_work) || exit 2 + "$nd" --work "$w33b" /usr/bin/touch "$w33b/ran" >/dev/null 2>&1 + r2=$? + [ "$r" = 125 ] && grep -q 'could not be shown to have ended' "$work/t33.out" && [ "$t33_child" = gone ] && + [ "$t33_kept" = yes ] && [ "$r2" -eq 0 ] && [ -e "$w33b/ran" ] + result "T-33 an ending that cannot be shown fails the run (125), ends the supervisor's own child and keeps the marker" $? \ + "(rc $r, command $t33_child, marker kept $t33_kept, next run rc $r2)" + + # T-35: a zombie outside (a child of this suite that has ended and is + # not reaped for 15 s) does not make the ending unprovable: it is found + # as a zombie and counts as dead from the next copy of the table on. + # On 980d6a5 before the fix: zombies were not found (they read as + # vanished), so every copy stayed open and the run failed with 125. + python3 -I -c 'import subprocess, sys, time +p = subprocess.Popen(["/usr/bin/true"]) +open(sys.argv[1], "w").write(str(p.pid)) +time.sleep(15) +p.wait()' "$work/t35.zombie" & + t35holder=$! + for _ in $(seq 1 50); do regular "$work/t35.zombie" && break; sleep 0.1; done + sleep 0.5 + t35z=$(tr -cd '0-9' <"$work/t35.zombie" 2>/dev/null) + t35_state=$(ps -o stat= -p "${t35z:-0}" 2>/dev/null | tr -d ' ') + w35=$(new_work) || exit 2 + "$nd" --work "$w35" python3 -I -B -c "$t24_leaver" "$w35" "t35-$nonce" ok >"$work/t35.out" 2>&1 + r=$? + sleep 0.5 + left=$(t24_left "$w35" "t35-$nonce") + t24_clean "$w35" "t35-$nonce" + wait "$t35holder" + echo "INFO T-35 zombie ${t35z:-?} state ${t35_state:-gone} during the run; rc $r; survivors:${left:- none}" + case $t35_state in Z*) t35_zombie=yes ;; *) t35_zombie=no ;; esac + [ "$t35_zombie" = yes ] && [ "$r" -eq 0 ] && [ -z "$left" ] + result "T-35 a zombie outside does not make the ending of a run unprovable" $? "(zombie $t35_zombie, rc $r)" + + # T-34: TERM to no-docker.sh while the command is starting processes + # (at 0, 0.15 and 0.4 s after it recorded its first one; it starts one + # every 50 ms for 1 s): processes must have been recorded when TERM is + # sent; each run ends within 15 s, interrupted (143) or, if the ending + # could not be shown, 125 with that message; none of the processes it + # recorded survives once the run ended or was ended through its own + # marker directory. + t34_spawn=' +import os, sys, time +w, tag = sys.argv[1], sys.argv[2] +open(w + "/mark", "w").write(os.environ.get("NO_DOCKER_MARK", "")) +SLEEP = [sys.executable, "-I", "-B", "-c", "import os, sys, time; open(sys.argv[1], \"a\").write(f\"{os.getpid()}\\n\"); time.sleep(30)", w + "/pids", tag] +for _ in range(20): + if os.fork() == 0: + os.setsid() + os.execv(SLEEP[0], SLEEP) + time.sleep(0.05) +time.sleep(30) +' + t34_bad="" + for t34_delay in 0 0.15 0.4; do + w34=$(new_work) || exit 2 + t34_before=$(cd "$ndtmp" && ls -d no-docker-mark.* 2>/dev/null) + python3 -I -c 'import os, signal, sys +signal.signal(signal.SIGINT, signal.SIG_DFL) +os.execv(sys.argv[1], sys.argv[1:])' "$nd" --work "$w34" python3 -I -B -c "$t34_spawn" "$w34" "t34-$nonce" >"$work/t34.out" 2>&1 & + ndpid=$! + for _ in $(seq 1 200); do [ "$(count_in . "$w34/pids")" -ge 1 ] && break; sleep 0.05; done + sleep "$t34_delay" + t34_n=$(count_in . "$w34/pids") + t0=$(date +%s) + ps -o args= -p "$ndpid" 2>/dev/null | grep -q -- no-docker && kill -TERM "$ndpid" + wait "$ndpid" + r=$? + took=$(($(date +%s) - t0)) + t34_all=$(count_in . "$w34/pids") + sleep 0.5 + left=$(t24_left "$w34" "t34-$nonce") + if [ "$r" -eq 125 ]; then + # Not shown: the kept marker directory of this run ends what is left. + # It is the one the command recorded, if that is a directory that did + # not exist before this iteration and whose owner record names this + # run's no-docker.sh (now gone); --end also refuses a directory whose + # owner still runs. + t34_mark="" + regular "$w34/mark" && t34_mark=$(cat "$w34/mark") + case ${t34_mark#"$ndtmp"/} in no-docker-mark.*[!A-Za-z0-9_]* | "$t34_mark") t34_mark="" ;; no-docker-mark.?*) ;; *) t34_mark="" ;; esac + case " +$t34_before +" in *" +${t34_mark#"$ndtmp"/} +"*) t34_mark="" ;; esac + if [ -n "$t34_mark" ] && [ -d "$t34_mark" ] && [ ! -L "$t34_mark" ] && regular "$t34_mark/owner" && + [ "$(cut -d' ' -f1 "$t34_mark/owner")" = "$ndpid" ]; then + python3 -I "$root/scripts/ci/no-docker-probe.py" --end "$t34_mark" >/dev/null 2>&1 + fi + sleep 0.5 + left=$(t24_left "$w34" "t34-$nonce") + fi + t24_clean "$w34" "t34-$nonce" + echo "INFO T-34 after ${t34_delay}s: $t34_n recorded at TERM, $t34_all in all, rc $r, ${took}s, survivors:${left:- none}" + [ "$t34_n" -ge 1 ] || t34_bad="$t34_bad delay$t34_delay:nothing-recorded-at-TERM" + case $r in + 143) ;; + 125) grep -q 'could not be shown to have ended' "$work/t34.out" || t34_bad="$t34_bad delay$t34_delay:125-without-message" ;; + *) t34_bad="$t34_bad delay$t34_delay:rc$r" ;; + esac + [ "$took" -le 15 ] || t34_bad="$t34_bad delay$t34_delay:${took}s" + [ -z "$left" ] || t34_bad="$t34_bad delay$t34_delay:survivors$left" + done + [ -z "$t34_bad" ] + result "T-34 TERM while the command starts processes: every run ends in time and none of its processes survives" $? "(${t34_bad:- all ok})" + + # --- T-28 no debugger access to a process outside (A-M2). The canaries + # are copies of /bin/sleep in this suite's directory, signed ad hoc with + # get-task-allow, so that a debugger of the same user may attach to them; + # the control attaches to one from outside and reads the other's memory + # map. Inside, ptrace(PT_ATTACHEXC) on the other must fail (EPERM), its + # task name port and its memory map (vmmap) must be refused, and it must + # still run. + # On dfb6ee4: the profile denied none of these, so the attach from inside + # succeeded (and ended the canary when the attacher exited). + t28c=$work/t28-sleep + cp /bin/sleep "$t28c" && + printf '%s\n' '' \ + '' \ + 'com.apple.security.get-task-allow' >"$work/t28.plist" && + codesign -f -s - --entitlements "$work/t28.plist" "$t28c" >/dev/null 2>&1 || exit 2 + t28_attach=' +import ctypes, errno, subprocess, sys +libc = ctypes.CDLL(None, use_errno=True) +pid = int(sys.argv[1]) +if sys.argv[2] == "inside": + me = ctypes.c_uint.in_dll(libc, "mach_task_self_").value + port = ctypes.c_uint(0) + kr = libc.task_name_for_pid(me, pid, ctypes.byref(port)) + vm = subprocess.run(["/usr/bin/vmmap", "--summary", str(pid)], capture_output=True, timeout=60).returncode + print("task-name " + ("got" if kr == 0 else "refused") + " vmmap " + ("read" if vm == 0 else "refused"), end=" ") +r = libc.ptrace(14, pid, None, 0) # PT_ATTACHEXC +print("attached" if r == 0 else "denied " + errno.errorcode.get(ctypes.get_errno(), "?")) +' + "$t28c" 60 & + t28a=$! + "$t28c" 60 & + t28b=$! + sleep 0.3 + t28_vm=$(/usr/bin/vmmap --summary "$t28b" >/dev/null 2>&1 && echo read || echo refused) + t28_out=$(python3 -I -c "$t28_attach" "$t28a" outside 2>&1) + t28_in=$("$nd" python3 -I -c "$t28_attach" "$t28b" inside 2>/dev/null) + r=$? + t28_state=$(ps -o stat= -p "$t28b" 2>/dev/null | tr -d ' ') + for p in "$t28a" "$t28b"; do + ps -o args= -p "$p" 2>/dev/null | grep -q -- "$t28c" && kill -KILL "$p" 2>/dev/null + done + wait "$t28a" "$t28b" 2>/dev/null + [ "$t28_out" = attached ] && [ "$t28_vm" = read ] && [ "$r" -eq 0 ] && + [ "$t28_in" = "task-name refused vmmap refused denied EPERM" ] && case $t28_state in S*) true ;; *) false ;; esac + result "T-28 a debugger attach, task port or memory map of a process outside is refused from inside (control: they work from outside)" $? \ + "(outside: $t28_out, vmmap $t28_vm; inside rc $r: $t28_in; canary state ${t28_state:-gone})" + + # --- T-29 another terminal cannot be read from inside: a pseudo-terminal + # this case opens outside stands for one; its device is opened by path + # (control: outside it can be, and a line written to it is read). + # On dfb6ee4: the profile allowed reading /dev/ttys*, so the line typed + # into another terminal was read. + t29=$(python3 -I -c ' +import os, pty, select, subprocess, sys +probe = "import os, select, sys\ntry:\n fd = os.open(sys.argv[1], os.O_RDONLY | os.O_NOCTTY)\nexcept OSError as e:\n print(\"denied\", e.errno)\n sys.exit(0)\nr, _, _ = select.select([fd], [], [], 5)\nprint(\"read\" if r and os.read(fd, 64) else \"opened\")\n" +out = [] +for where in ("outside", "inside"): + master, slave = pty.openpty() + argv = [sys.executable, "-I", "-c", probe, os.ttyname(slave)] + if where == "inside": + argv = [sys.argv[1]] + argv + p = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + select.select([], [], [], 1.5) + os.write(master, b"typed\n") + try: + o, _ = p.communicate(timeout=60) + except subprocess.TimeoutExpired: + p.kill() + o, _ = p.communicate() + out.append(f"{where}:{o.strip()}") + os.close(master) + os.close(slave) +print(" ".join(out)) +' "$nd" 2>&1) + [ "$t29" = "outside:read inside:denied 1" ] + result "T-29 another terminal's device cannot be read from inside (control: it can from outside)" $? "($t29)" +else + echo "INFO T-06, T-08 to T-15, T-21, T-22, T-24 to T-35 skipped: macOS sandbox profile only" +fi + +# --- T-07 loopback ports listening before the isolation +cat >"$work/t07.py" <<'PY' +import socket, sys, time +s = socket.socket() +s.bind(("127.0.0.1", 0)) +s.listen(4) +open(sys.argv[1], "w").write(str(s.getsockname()[1])) +time.sleep(60) +PY +python3 -I "$work/t07.py" "$work/t07.port" & +t07pid=$! +for _ in 1 2 3 4 5 6 7 8 9 10; do [ -s "$work/t07.port" ] && break; sleep 0.2; done +# On Linux, no-docker.sh frames the output with ::stop-commands:: lines. +keep_stream out "$work/t07.out" "$work/t07.err" ndl python3 -I -c ' +import socket, sys +def reach(port): + s = socket.socket() + s.settimeout(2) + try: + s.connect(("127.0.0.1", port)) + return "reached" + except OSError: + return "refused" + finally: + s.close() +own = socket.socket() +own.bind(("127.0.0.1", 0)) +own.listen(1) +print(reach(int(sys.argv[1])), reach(own.getsockname()[1])) +' "$(cat "$work/t07.port" 2>/dev/null || echo 0)" +t07_rc=$ks_rc +kill "$t07pid" 2>/dev/null +wait "$t07pid" 2>/dev/null +t07_verdict "$t07_rc" "$work/t07.out" && [ -s "$work/t07.port" ] +result "T-07 a loopback port listening before the isolation is unreachable, one opened inside it is reachable" $? "(rc $t07_rc: $t07)" + +if [ "$os" = Linux ]; then + echo "INFO mount: $(mount --version 2>&1 | head -n 1)" + echo "INFO mounts outside the isolation:" + findmnt -rn -o TARGET,FSTYPE,OPTIONS 2>&1 | sed 's/^/INFO /' + echo "INFO mounts inside the isolation (and the probe's report):" + ndl findmnt -rn -o TARGET,FSTYPE,OPTIONS 2>&1 | sed 's/^/INFO /' + + # --- L-tests: the kernel-enforced isolation (Linux CI only). Each case + # prints a fixed number of result lines (97 in all with L-16 at the end) + # and has its control: a denial counts only where the same target is + # shown reachable outside. Sockets, carriers and mounts here are this + # suite's own and are ended or undone in cleanup. + profiles() { sudo -n cat /sys/kernel/security/apparmor/profiles; } + # listed NAME-PREFIX: the number of loaded profiles whose name starts so. + listed() { + local l + l=$(profiles 2>/dev/null) || { + echo unknown + return + } + printf '%s\n' "$l" | grep -c "^$1" + } + ctl_dirs() { find "$RUNNER_TEMP" -maxdepth 1 -name 'no-docker-ctl.*' 2>/dev/null | wc -l | tr -d ' '; } + # The listeners and the client the cases use (outside and inside). + cat >"$ro/lsock.py" <<'PY' +import os, socket, sys, time +kind, path, ready = sys.argv[1:4] +fam = socket.SOCK_DGRAM if kind == "dgram" else socket.SOCK_STREAM +s = socket.socket(socket.AF_UNIX, fam) +s.bind("\0" + path if kind == "abstract" else path) +if kind != "dgram": + s.listen(64) +with open(ready + ".tmp", "w") as f: + f.write(f"{os.getpid()} {time.time_ns()}") +os.rename(ready + ".tmp", ready) +count = 0 +end = time.time() + 900 +while time.time() < end: + if kind == "dgram": + s.recv(65536) + else: + c = s.accept()[0] + if kind == "scm": + fd = os.open(sys.argv[4], os.O_RDONLY) + socket.send_fds(c, [b"x"], [fd]) + os.close(fd) + c.close() + count += 1 + with open(ready + ".count.tmp", "w") as f: + f.write(str(count)) + os.rename(ready + ".count.tmp", ready + ".count") +PY + cat >"$ro/lclient.py" <<'PY' +import errno, os, socket, sys, time +def name(e): + return "timeout" if isinstance(e, socket.timeout) else errno.errorcode.get(e.errno, str(e.errno)) +def unix(kind): + s = socket.socket(socket.AF_UNIX, kind) + opened.append(s) + return s +args = sys.argv[1:] +for op, target in zip(args[0::2], args[1::2]): + opened = [] + try: + if op == "stream": + s = unix(socket.SOCK_STREAM) + s.settimeout(2) + s.connect(target) + r = "OK" + elif op == "dgram-connect": + unix(socket.SOCK_DGRAM).connect(target) + r = "OK" + elif op == "dgram-send": + unix(socket.SOCK_DGRAM).sendto(b"x", target) + r = "OK" + elif op == "abstract": + s = unix(socket.SOCK_STREAM) + s.settimeout(2) + s.connect("\0" + target) + r = "OK" + elif op == "scm": + s = unix(socket.SOCK_STREAM) + s.settimeout(2) + s.connect(target) + fds = socket.recv_fds(s, 16, 4)[1] + for fd in fds: + os.close(fd) + r = f"OK fds={len(fds)}" + elif op == "vsock": + opened.append(socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)) + r = "OK" + elif op == "late": + # The listener writes "ready" next to its socket once it listens. + ready = os.path.join(os.path.dirname(target), "ready") + for _ in range(600): + if os.path.exists(ready): + break + time.sleep(0.05) + print(f"TIME {time.time_ns()}", flush=True) + s = unix(socket.SOCK_STREAM) + s.settimeout(2) + s.connect(target) + r = "OK" + elif op == "label": + with open("/proc/self/attr/apparmor/current") as f: + r = f.read().strip() + print(f"PROFILE {os.environ.get('NO_DOCKER_PROFILE', '')}", flush=True) + elif op == "fds": + open_fds = [] + for fd in sorted(int(x) for x in os.listdir("/proc/self/fd")): + try: + os.fstat(fd) + open_fds.append(str(fd)) + except OSError: + pass + r = " ".join(open_fds) + else: + r = "unknown-op" + except OSError as e: + r = name(e) + finally: + for s in opened: + s.close() + print(f"RESULT {op} {target} {r}", flush=True) +PY + # res FILE OP TARGET: the result the client printed for OP on TARGET. + res() { sed -n "s|^RESULT $2 $3 ||p" "$1" 2>/dev/null | tail -n 1; } + # lsock KIND PATH READY [ARG]: start a listener of this suite and wait + # until it is ready; its pid is the first word of READY. + lsock() { + python3 -I "$ro/lsock.py" "$@" /dev/null 2>&1 & + for _ in $(seq 1 80); do [ -s "$3" ] && break; sleep 0.1; done + [ -s "$3" ] || return 1 + printf '%s\n' "$3" >>"$work/l-listeners" + } + # lstop READY: end the listener recorded there (by its pid, then forgotten). + lstop() { + local p + p=$(cut -d' ' -f1 "$1" 2>/dev/null) + case $p in "" | *[!0-9]*) return ;; esac + kill "$p" 2>/dev/null + for _ in $(seq 1 20); do kill -0 "$p" 2>/dev/null || break; sleep 0.1; done + rm -f "$1" + } + # wait_bounded PID SECONDS: wait for a child of this suite; 124 if it is + # still running after SECONDS (then it is killed). + wait_bounded() { + local i=0 + while kill -0 "$1" 2>/dev/null; do + if [ "$i" -ge $(($2 * 10)) ]; then + kill -KILL "$1" 2>/dev/null + wait "$1" 2>/dev/null + return 124 + fi + sleep 0.1 + i=$((i + 1)) + done + wait "$1" + } + # left_loaded LOG: release a profile a launch reported left loaded, once + # its namespace's processes are gone (RR-06); recorded for cleanup too. + left_loaded() { + local line n c + line=$(grep -m1 '^no-docker: refused: profile .* still loaded' "$1" 2>/dev/null) || return 1 + n=${line#no-docker: refused: profile } + n=${n%% *} + c=${line##*control directory } + c=${c%% is kept} + printf '%s %s\n' "$n" "$c" >>"$work/l-release" + for _ in $(seq 1 20); do + # shellcheck disable=SC2024 # this shell appends the helper's messages to the log + if sudo -n timeout 30 /usr/bin/python3 -I "$root/scripts/ci/no-docker-aa.py" release "$n" "$c" >>"$1" 2>&1; then + rm -rf "$c" + return 0 + fi + sleep 2 + done + return 1 + } + nd_listed_before=$(listed flowcrm-nd-) + + + # --- T-16 a canary inside the isolation: private pid view, read-only + # $HOME/workspace/RUNNER_TEMP, private /tmp, read-only mounts, masked + # cgroups, private IPC, loopback only; confined by exactly its profile, + # descriptors 0 to 2 only, the launch's fence refused; and every host + # socket gathered outside by this suite classified: (i) reachable outside + # and visible inside must be refused with EACCES exactly; (ii) under a + # directory the isolation hides (/run, /var/run, /tmp, /var/tmp, /dev/shm) + # is not found inside; (iii) not reachable outside is only reported; none + # is ever reached inside. A socket of this suite in $RUNNER_TEMP makes + # sure class (i) is not empty. + # On 38b5ebd: no pid namespace (pid 1 is the runner's systemd, foreign + # processes visible), and $HOME and the workspace were writable. + cat >"$ro/t16.py" <<'PY' +import errno, glob, os, socket, sys +ipc_host, pid_host, out_tmp, out_shm, in_tmp, classes, nonce = sys.argv[1:8] +targets = sys.argv[8:] +me = str(os.getuid()) +def say(good, text): + print(("t16 ok " if good else "t16 bad ") + text) +def status(pid): + st = {} + with open(f"/proc/{pid}/status") as f: + for line in f: + k, _, v = line.partition(":") + st[k] = v.strip() + return st +def reach(path): + for kind in (socket.SOCK_STREAM, socket.SOCK_DGRAM): + s = socket.socket(socket.AF_UNIX, kind) + s.settimeout(2) + try: + s.connect(path) + return "OK" + except socket.timeout: + return "timeout" + except OSError as e: + if e.errno == errno.EPROTOTYPE: + continue + return errno.errorcode.get(e.errno, str(e.errno)) + finally: + s.close() + return "EPROTOTYPE" +st = status("self") +say(st.get("NSpid", "").split() == [str(os.getpid())], f"NSpid {st.get('NSpid')} is this pid alone (a /proc of its own pid namespace)") +own = os.readlink("/proc/self/ns/pid") +say(pid_host == "pid:[4026531836]" and own not in (pid_host, "pid:[4026531836]"), + f"a pid namespace below the caller's, the initial one ({own}; caller {pid_host})") +s1 = status(1) +say(s1["Uid"].split()[0] == "0" and s1["Name"] == "sh", f"pid 1 is the isolation's root shell ({s1['Name']}, uid {s1['Uid'].split()[0]})") +others = [] +for d in os.listdir("/proc"): + if d.isdigit() and d != "1": + try: + s = status(d) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESRCH): + others.append(f"{d}:unreadable") + continue + if any(u != me for u in s["Uid"].split()) or s["Name"] in ("systemd", "Runner.Worker", "Runner.Listener", "dockerd", "containerd"): + others.append(f"{d}:{s['Name']}") +say(not others, f"no foreign process visible ({' '.join(others) or 'none'})") +with open("/proc/self/mountinfo") as f: + procs = [l.split() for l in f if l.split()[4] == "/proc"] +hidden = [o for r in procs for o in r[5].split(",") + r[r.index("-") + 3].split(",") + if (o.startswith("hidepid=") and o not in ("hidepid=0", "hidepid=off")) or o.startswith("subset=")] +say(len(procs) >= 1 and not hidden, f"/proc hides no process ({','.join(hidden) or 'no hidepid or subset'})") +try: + os.listdir("/proc/1/root") + say(False, "/proc/1/root is not readable") +except OSError as e: + say(e.errno in (errno.EACCES, errno.EPERM), f"/proc/1/root is not readable ({errno.errorcode.get(e.errno)})") +with open(classes) as f: + rows = [line.rstrip("\n").split(" ", 2) for line in f if line.strip()] +for cls, outside, path in rows: + got = reach(path) + if cls == "i": + good = got == "EACCES" + elif cls == "ii": + good = got in ("ENOENT", "EACCES") + else: + good = got != "OK" + say(good, f"host socket class ({cls}) {path}: outside {outside}, inside {got}") +for t in targets: + p = os.path.join(t, f".flowcrm-no-docker-test-t16-{nonce}") + try: + os.close(os.open(p, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)) + os.unlink(p) + say(False, f"write in {t} fails (written, removed again)") + except OSError as e: + say(e.errno == errno.EROFS, f"write in {t} fails with EROFS ({errno.errorcode.get(e.errno)})") +say(not os.path.exists(out_tmp), "a file in the caller's /tmp is not visible") +say(not os.path.exists(out_shm), "a file in the caller's /dev/shm is not visible") +try: + os.close(os.open(in_tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)) + say(True, "the private /tmp is writable") +except OSError as e: + say(False, f"the private /tmp is writable ({e})") +allowed = {"/tmp", "/var/tmp", "/dev/shm", "/run", "/var/run", "/dev/mqueue", "/proc", "/dev", "/dev/pts", + os.environ.get("NO_DOCKER_RUN_DIR", "")} +rw = [] +cgroup_fs = "none" +with open("/proc/self/mountinfo") as f: + rows = [line.split() for line in f] +for r in rows: + point = r[4].replace("\\040", " ") + try: + fd = os.open(point, os.O_PATH | os.O_NOFOLLOW) + except OSError: + continue + try: + mnt = [l.split()[1] for l in open(f"/proc/self/fdinfo/{fd}") if l.startswith("mnt_id:")][0] + finally: + os.close(fd) + if mnt != r[0]: + continue + if point == "/sys/fs/cgroup": + cgroup_fs = r[r.index("-") + 1] + if point not in allowed and "ro" not in r[5].split(","): + rw.append(point) +say(not rw, f"every other reachable mount is read-only ({' '.join(rw) or 'none writable'})") +say(cgroup_fs != "cgroup2" and not os.access("/sys/fs/cgroup/cgroup.procs", os.W_OK) + and not os.access("/sys/fs/cgroup/cgroup.kill", os.W_OK), f"/sys/fs/cgroup is masked ({cgroup_fs})") +say(os.readlink("/proc/self/ns/ipc") != ipc_host, f"private IPC namespace ({os.readlink('/proc/self/ns/ipc')}, caller {ipc_host})") +names = sorted(n for _, n in socket.if_nameindex()) +say(names == ["lo"], f"only loopback ({' '.join(names)})") +with open("/proc/self/attr/apparmor/current") as f: + label = f.read().strip() +profile = os.environ.get("NO_DOCKER_PROFILE", "") +say(bool(profile) and label == f"{profile} (enforce)", f"confined by exactly the launch's profile ({label})") +fds = [] +for fd in sorted(int(x) for x in os.listdir("/proc/self/fd")): + try: + os.fstat(fd) + fds.append(fd) + except OSError: + pass +say(fds == [0, 1, 2], f"only descriptors 0 to 2 are open ({' '.join(map(str, fds))})") +fence = glob.glob(os.path.join(os.environ.get("HOME", "/nonexistent"), "work/_temp/no-docker-fence.*")) +got = [reach(os.path.join(fence[0], k)) for k in ("stream", "dgram")] if len(fence) == 1 else [f"{len(fence)} fence directories"] +say(got == ["EACCES", "EACCES"], f"the launch's fence refuses the command too ({' '.join(got)})") +PY + t16_tmp=/tmp/.flowcrm-no-docker-test-t16-$nonce + t16_shm=/dev/shm/.flowcrm-no-docker-test-t16-$nonce + t16_in=/tmp/.flowcrm-no-docker-test-t16-in-$nonce + : >"$t16_tmp" + : 2>/dev/null >"$t16_shm" + # Sockets of the host, found here independently of no-docker.sh, and one + # of this suite where nothing hides it. + mkdir -m 700 "$ro/t16own" || exit 2 + lsock stream "$ro/t16own/s" "$work/t16own.ready" + { python3 -I -c ' +import sys +with open("/proc/net/unix", "rb") as f: + next(f) + for line in f: + p = line.split(None, 7) + if len(p) == 8 and p[7].startswith(b"/"): + sys.stdout.buffer.write(p[7].rstrip(b"\n") + b"\0") +' + find "$home" -xdev -type s -print0 2>/dev/null + printf '%s\0' "$ro/t16own/s" + } | sort -zu >"$ro/t16-sockets" + # Classify each outside: reachable here, and under a directory the + # isolation hides. + python3 -I -c ' +import errno, socket, sys +masked = ("/run/", "/var/run/", "/tmp/", "/var/tmp/", "/dev/shm/") +for raw in open(sys.argv[1], "rb").read().split(b"\0"): + if not raw or b" " in raw or b"\n" in raw: + continue + path = raw.decode("utf-8", "replace") + got = "EPROTOTYPE" + for kind in (socket.SOCK_STREAM, socket.SOCK_DGRAM): + s = socket.socket(socket.AF_UNIX, kind) + s.settimeout(2) + try: + s.connect(path) + got = "OK" + break + except socket.timeout: + got = "timeout" + break + except OSError as e: + got = errno.errorcode.get(e.errno, str(e.errno)) + if e.errno != errno.EPROTOTYPE: + break + finally: + s.close() + cls = "iii" if got != "OK" else "ii" if path.startswith(masked) else "i" + print(cls, got, path) +' "$ro/t16-sockets" >"$ro/t16-classes" + t16_targets=("$home") + for v in GITHUB_WORKSPACE RUNNER_TEMP; do [ -z "${!v-}" ] || t16_targets+=("${!v}"); done + [ ! -d /opt/hostedtoolcache ] || t16_targets+=(/opt/hostedtoolcache) + cp "$ro/t16-sockets" "$ro/t16-classes" "$work/" || exit 2 + keep_stream both "$work/t16.out" ndl python3 -I "$ro/t16.py" "$(readlink /proc/self/ns/ipc)" "$(readlink /proc/self/ns/pid)" "$t16_tmp" "$t16_shm" "$t16_in" \ + "$ro/t16-classes" "$nonce" "${t16_targets[@]}" + r=$ks_rc + t16=$(cat "$work/t16.out") + lstop "$work/t16own.ready" + printf '%s\n' "$t16" | grep '^t16 ' | sed 's/^/INFO /' + t16_left="" + for t in "${t16_targets[@]}"; do + if [ -e "$t/.flowcrm-no-docker-test-t16-$nonce" ]; then + t16_left="$t16_left $t" + rm -f "$t/.flowcrm-no-docker-test-t16-$nonce" + fi + done + [ ! -e "$t16_in" ] || t16_left="$t16_left /tmp" + t16_ok=$(printf '%s\n' "$t16" | grep -c '^t16 ok') + t16_n=$(grep -c . "$ro/t16-classes") + t16_i=$(grep -c '^i ' "$ro/t16-classes") + # One line per check: 13, the label, the descriptors and the fence (3), + # one per protected directory and one per host socket. + t16_want=$((16 + ${#t16_targets[@]} + t16_n)) + [ "$r" -eq 0 ] && ! printf '%s\n' "$t16" | grep -q '^t16 bad' && [ "$t16_ok" -eq "$t16_want" ] && [ -z "$t16_left" ] && + [ "$t16_i" -ge 1 ] + result "T-16 inside the isolation: private pids, confined, host sockets refused, read-only \$HOME/workspace/RUNNER_TEMP, private /tmp, read-only mounts, masked cgroups, private IPC" $? \ + "(rc $r, $t16_ok of $t16_want ok, $t16_n host sockets, $t16_i of class (i), markers left in:${t16_left:- none})" + rm -f "$t16_tmp" "$t16_shm" "$t16_in" + + # --- T-17 no process survives the isolated command: a `setsid -f` canary + # with a unique marker, after a normal end and after TERM delivered to + # no-docker.sh itself (as by Ctrl-C or a caller forwarding the signal; a + # cancelled GitHub step does not signal no-docker.sh and is not covered). The + # canary is a shell kept alive by "sleep 60; :", so the process with the + # marker in its command line is the one that must die; the command checks + # that it runs before it returns. Checked from outside, after + # no-docker.sh returned. + # On 38b5ebd: no pid namespace and no --kill-child, so the canary + # survives; and TERM to no-docker.sh left sudo, unshare and the command + # running. + t17m=t17-$nonce + keep_stream out "$work/t17.out" "$work/t17.err" ndl /bin/sh -c 'setsid -f /bin/sh -c "sleep 60; :" "$1-in" /dev/null 2>&1 +for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do + if pgrep -f -- "$1-in" >/dev/null; then echo canary-started; exit 0; fi + sleep 0.1 +done +echo canary-missing' _ "$t17m" + t17=$(grep -v '^::' "$work/t17.out") + for _ in 1 2 3 4 5 6 7 8 9 10; do pgrep -f -- "$t17m-in" >/dev/null || break; sleep 0.5; done + pgrep -f -- "$t17m-in" >/dev/null + left=$? + pkill -TERM -f -- "$t17m-in" 2>/dev/null + [ "$t17" = canary-started ] && [ "$left" -ne 0 ] + result "T-17 a daemonised process does not outlive the isolated command" $? \ + "($t17, survivor $([ "$left" -eq 0 ] && echo yes || echo no))" + + t17c=t17c-$nonce + count_launch + "$nd" /bin/sh -c 'setsid -f /bin/sh -c "sleep 60; :" "$1-in" /dev/null 2>&1; sleep 60; :' _ "$t17c" >"$work/t17c.out" 2>&1 & + t17pid=$! + # Up to 150 s: longer than the bound of the cleanup (112 s). + for _ in $(seq 1 300); do pgrep -f -- "$t17c-in" >/dev/null && break; sleep 0.5; done + started=no + pgrep -f -- "$t17c-in" >/dev/null && started=yes + kill -TERM "$t17pid" 2>/dev/null + wait "$t17pid" 2>/dev/null + r=$? + for _ in 1 2 3 4 5 6 7 8 9 10; do pgrep -f -- "$t17c-in" >/dev/null || break; sleep 0.5; done + pgrep -f -- "$t17c-in" >/dev/null + left=$? + pkill -TERM -f -- "$t17c" 2>/dev/null + [ "$started" = yes ] && [ "$left" -ne 0 ] && [ "$r" -eq 143 ] + result "T-17 TERM delivered to no-docker.sh itself kills every process inside the isolation (143)" $? \ + "(canary started $started, rc $r, survivor $([ "$left" -eq 0 ] && echo yes || echo no))" + + # --- T-18 unsupported use is refused: uid 0 (a real sudo run) and a Linux + # that is not a GitHub-hosted runner. The command is `true`. + # On 38b5ebd: neither was checked, so every case ran `true` (rc 0). + # The command's stdout is kept in $work/t18-N.out and its stderr in + # $work/t18-N.err (N counts the calls). + t18n=0 + t18() { # name expected-message command... + local name=$1 msg=$2 err + shift 2 + t18n=$((t18n + 1)) + keep_stream err "$work/t18-$t18n.err" "$work/t18-$t18n.out" "$@" + r=$ks_rc + err=$(cat "$work/t18-$t18n.err") + [ "$r" -eq 125 ] && printf '%s' "$err" | grep -q -- "$msg" + result "T-18 $name: refused" $? "(rc $r)" + } + t18 "uid 0 (sudo)" "running as root" sudo -n "$nd" /usr/bin/true + t18 "RUNNER_ENVIRONMENT unset" "GitHub-hosted" env -u RUNNER_ENVIRONMENT "$nd" /usr/bin/true + t18 "RUNNER_ENVIRONMENT=self-hosted" "GitHub-hosted" env RUNNER_ENVIRONMENT=self-hosted "$nd" /usr/bin/true + t18 "GITHUB_ACTIONS unset" "GitHub-hosted" env -u GITHUB_ACTIONS "$nd" /usr/bin/true + t18 "GITHUB_WORKSPACE outside \$HOME" "GitHub-hosted" env GITHUB_WORKSPACE=/var/tmp "$nd" /usr/bin/true + # Without a Runner.Worker ancestor: started through setsid -f, so it is + # reparented away from this job's process tree; its status comes back in + # a file. + setsid -f /bin/sh -c '"$1" /usr/bin/true >"$2.out" 2>"$2.err"; echo $? >"$2.tmp"; mv "$2.tmp" "$2"' _ "$nd" "$work/t18.rc" + for _ in $(seq 1 240); do [ -s "$work/t18.rc" ] && break; sleep 0.5; done + r=$(cat "$work/t18.rc" 2>/dev/null) + [ "$r" = 125 ] && grep -q "no Runner.Worker ancestor" "$work/t18.rc.err" 2>/dev/null + result "T-18 no Runner.Worker ancestor: refused" $? "(rc ${r:-none})" + t18 "--negative-control off a GitHub-hosted runner" "GitHub-hosted" env RUNNER_ENVIRONMENT=self-hosted "$nd" --negative-control /usr/bin/true + t18 "--negative-control in a nested run" "refused inside the isolation" env NO_DOCKER_ACTIVE=1 "$nd" --negative-control /usr/bin/true + + # --- L-01 host pathname stream, datagram connect and sendto, abstract: + # refused inside, reachable outside before and after. + mkdir -m 700 "$ro/l01" || exit 2 + l01_ok=yes + lsock stream "$ro/l01/s" "$work/l01s.ready" && lsock dgram "$ro/l01/d" "$work/l01d.ready" && + lsock abstract "flowcrm-l01-$nonce" "$work/l01a.ready" || l01_ok=no + l01_targets=(stream "$ro/l01/s" dgram-connect "$ro/l01/d" dgram-send "$ro/l01/d" abstract "flowcrm-l01-$nonce") + python3 -I "$ro/lclient.py" "${l01_targets[@]}" >"$work/l01.before" 2>&1 + ndl python3 -I "$ro/lclient.py" "${l01_targets[@]}" label - >"$work/l01.in" 2>"$work/l01.err" + r=$? + python3 -I "$ro/lclient.py" "${l01_targets[@]}" >"$work/l01.after" 2>&1 + for t in "stream $ro/l01/s EACCES" "dgram-connect $ro/l01/d EACCES" "dgram-send $ro/l01/d EACCES" \ + "abstract flowcrm-l01-$nonce ECONNREFUSED"; do + read -r op tgt want <<<"$t" + [ "$l01_ok" = yes ] && [ "$r" -eq 0 ] && [ "$(res "$work/l01.in" "$op" "$tgt")" = "$want" ] + result "L-01 inside: $op to a socket of the host is refused with $want" $? "(rc $r, got $(res "$work/l01.in" "$op" "$tgt"))" + done + l01_profile=$(sed -n 's/^PROFILE //p' "$work/l01.in") + case $l01_profile in flowcrm-nd-????????????????????????????????) ;; *) l01_profile=none ;; esac + [ "$(res "$work/l01.in" label -)" = "$l01_profile (enforce)" ] + result "L-01 inside: the command is confined by exactly its profile in enforce mode" $? "($(res "$work/l01.in" label -))" + for when in before after; do + for t in "stream $ro/l01/s" "dgram-connect $ro/l01/d" "dgram-send $ro/l01/d" "abstract flowcrm-l01-$nonce"; do + read -r op tgt <<<"$t" + [ "$(res "$work/l01.$when" "$op" "$tgt")" = OK ] + result "L-01 control $when: $op outside reaches the same socket" $? "($(res "$work/l01.$when" "$op" "$tgt"))" + done + done + lstop "$work/l01s.ready" + lstop "$work/l01d.ready" + lstop "$work/l01a.ready" + + # --- L-02 a bind-mounted alias of a host socket's directory. + mkdir -m 700 "$ro/l02src" "$ro/l02alias" || exit 2 + lsock stream "$ro/l02src/s" "$work/l02.ready" && sudo -n mount --bind "$ro/l02src" "$ro/l02alias" && l02_mounted=yes + python3 -I "$ro/lclient.py" stream "$ro/l02alias/s" >"$work/l02.out" 2>&1 + [ "${l02_mounted-}" = yes ] && [ "$(res "$work/l02.out" stream "$ro/l02alias/s")" = OK ] + result "L-02 control: outside, the socket is reachable through the alias" $? + ndl python3 -I "$ro/lclient.py" stream "$ro/l02alias/s" >"$work/l02.in" 2>"$work/l02.err" + r=$? + [ "$(res "$work/l02.in" stream "$ro/l02alias/s")" = EACCES ] + result "L-02 inside: a connect through the alias is refused with EACCES" $? "($(res "$work/l02.in" stream "$ro/l02alias/s"))" + [ "$r" -eq 0 ] + result "L-02 the run proceeds (the command ran)" $? "(rc $r)" + sudo -n umount "$ro/l02alias" && l02_mounted="" + lstop "$work/l02.ready" + + # --- L-03 a directory that cannot be listed (0311): N1 a listener in the + # host network namespace, N2 in one of its own, N0 the socket file of a + # listener that ended. + l03() { # n1|n2|n0 listener-prefix...: start the listener of a case in a 0311 directory + local n=$1 + shift + mkdir -m 700 "$ro/l03$n" || return 1 + "$@" python3 -I "$ro/lsock.py" stream "$ro/l03$n/s" "$work/l03$n.ready" /dev/null 2>&1 & + for _ in $(seq 1 80); do [ -s "$work/l03$n.ready" ] && break; sleep 0.1; done + [ -s "$work/l03$n.ready" ] && printf '%s\n' "$work/l03$n.ready" >>"$work/l-listeners" && chmod 311 "$ro/l03$n" + } + l03 n1 + l03 n2 sudo -n unshare --net -- setpriv --reuid "$(id -u)" --regid "$(id -g)" --clear-groups -- + l03 n0 && lstop "$work/l03n0.ready" + for n in n1:N1 n2:N2; do + case=${n#*:} n=${n%%:*} + python3 -I "$ro/lclient.py" stream "$ro/l03$n/s" >"$work/l03$n.out" 2>&1 + ndl python3 -I "$ro/lclient.py" stream "$ro/l03$n/s" >"$work/l03$n.in" 2>"$work/l03$n.err" + r=$? + [ "$(res "$work/l03$n.out" stream "$ro/l03$n/s")" = OK ] + result "L-03 $case control: outside, the socket in the unlistable directory is reachable" $? + [ "$r" -eq 0 ] && [ "$(res "$work/l03$n.in" stream "$ro/l03$n/s")" = EACCES ] + result "L-03 $case inside: the run proceeds and the connect is refused with EACCES" $? \ + "(rc $r, $(res "$work/l03$n.in" stream "$ro/l03$n/s"))" + done + python3 -I "$ro/lclient.py" stream "$ro/l03n0/s" >"$work/l03n0.out" 2>&1 + ndl python3 -I "$ro/lclient.py" stream "$ro/l03n0/s" >"$work/l03n0.in" 2>"$work/l03n0.err" + r=$? + [ "$r" -eq 0 ] && [ "$(res "$work/l03n0.in" stream "$ro/l03n0/s")" = ECONNREFUSED ] && + [ "$(res "$work/l03n0.out" stream "$ro/l03n0/s")" = ECONNREFUSED ] + result "L-03 N0: a socket file without a listener is ECONNREFUSED inside and outside" $? \ + "(rc $r, inside $(res "$work/l03n0.in" stream "$ro/l03n0/s"))" + lstop "$work/l03n1.ready" + lstop "$work/l03n2.ready" + chmod 700 "$ro/l03n1" "$ro/l03n2" "$ro/l03n0" 2>/dev/null + + # --- L-04 a socket bound outside after the command started. + mkdir -m 700 "$ro/l04" || exit 2 + g=$(new_work) || exit 2 + count_launch + "$nd" --work "$g" python3 -I "$ro/lclient.py" late "$ro/l04/s" >"$work/l04.in" 2>"$work/l04.err" & + l04pid=$! + for _ in $(seq 1 300); do [ -e "$g/started" ] && break; sleep 0.1; done + l04_started=$(python3 -I -c 'import os, sys; print(os.stat(sys.argv[1]).st_mtime_ns)' "$g/started" 2>/dev/null || echo 0) + lsock stream "$ro/l04/s" "$ro/l04/ready" + wait_bounded "$l04pid" 150 + r=$? + python3 -I "$ro/lclient.py" stream "$ro/l04/s" >"$work/l04.out" 2>&1 + [ "$(res "$work/l04.out" stream "$ro/l04/s")" = OK ] + result "L-04 control: outside, the late socket is reachable" $? + [ "$r" -eq 0 ] && [ "$(res "$work/l04.in" late "$ro/l04/s")" = EACCES ] + result "L-04 inside: a socket bound after the start is refused with EACCES" $? "(rc $r, $(res "$work/l04.in" late "$ro/l04/s"))" + l04_bound=$(cut -d' ' -f2 "$ro/l04/ready" 2>/dev/null) + l04_tried=$(sed -n 's/^TIME //p' "$work/l04.in") + [ "${l04_started:-0}" -gt 0 ] && [ "${l04_bound:-0}" -gt "$l04_started" ] && [ "${l04_tried:-0}" -gt "${l04_bound:-0}" ] + result "L-04 the order: the command started, then the socket was bound, then the connect" $? \ + "(started $l04_started, bound ${l04_bound:-?}, connect ${l04_tried:-?})" + lstop "$ro/l04/ready" + + # --- L-05 sockets of the command's own still work. + ndl python3 -I -c ' +import socket +for kind, name in ((socket.SOCK_STREAM, "stream"), (socket.SOCK_DGRAM, "dgram")): + path = f"/tmp/l05-{name}" + a = socket.socket(socket.AF_UNIX, kind) + a.bind(path) + b = socket.socket(socket.AF_UNIX, kind) + try: + if kind == socket.SOCK_STREAM: + a.listen(1) + b.settimeout(2) + b.connect(path) + else: + b.sendto(b"x", path) + a.settimeout(2) + a.recv(10) + print(f"RESULT pair {name} OK") + except OSError as e: + print(f"RESULT pair {name} {e}") +' >"$work/l05.in" 2>"$work/l05.err" + r=$? + grep -q '^no-docker: ok a socket bound inside the isolation connects (ok)$' "$work/l05.err" + result "L-05 the gate's twin socket inside connected" $? + [ "$r" -eq 0 ] && [ "$(res "$work/l05.in" pair stream)" = OK ] + result "L-05 a stream socket pair of the command in /tmp connects" $? "($(res "$work/l05.in" pair stream))" + [ "$(res "$work/l05.in" pair dgram)" = OK ] + result "L-05 a datagram socket pair of the command in /tmp exchanges a message" $? "($(res "$work/l05.in" pair dgram))" + + # --- L-06 exec and children, escapes, signal dispositions, vsock. The + # command is /bin/sh, which reads its own status with builtins (a CPython + # would ignore SIGPIPE and SIGXFSZ itself). + ndl /bin/sh -c ' +while read -r k v rest; do + case $k in SigIgn:) echo "SIGIGN $v" ;; SigBlk:) echo "SIGBLK $v" ;; esac +done /proc/self/attr/apparmor/current 2>/dev/null; then echo "CHANGEPROFILE CHANGED"; else echo "CHANGEPROFILE DENIED"; fi +if aa-exec -p unconfined -- true 2>/dev/null; then echo "AAEXEC CHANGED"; else echo "AAEXEC DENIED"; fi +if unshare -U true 2>/dev/null; then echo "USERNS CREATED"; else echo "USERNS DENIED"; fi +python3 -I "$1" vsock - +echo "PROFILE $NO_DOCKER_PROFILE" +' l06 "$ro/lclient.py" >"$work/l06.in" 2>"$work/l06.err" + l06p=$(sed -n 's/^PROFILE //p' "$work/l06.in") + [ "$(sed -n 's/^GRANDCHILD //p' "$work/l06.in")" = "$l06p (enforce)" ] && [ -n "$l06p" ] + result "L-06 a grandchild of the command carries exactly the profile" $? "($(sed -n 's/^GRANDCHILD //p' "$work/l06.in"))" + l06pid=$(sed -n 's/^PID //p' "$work/l06.in") + grep -q "^no-docker: info the gate passed in [0-9.]*s (pid $l06pid)\$" "$work/l06.err" && [ -n "$l06pid" ] + result "L-06 the command runs in the gate's own process" $? "(command pid ${l06pid:-?})" + for t in "CHANGEPROFILE change_profile to unconfined" "AAEXEC aa-exec -p unconfined" "USERNS unshare -U"; do + read -r tag what <<<"$t" + grep -qx "$tag DENIED" "$work/l06.in" + result "L-06 inside: $what is denied" $? + done + grep -qx 'SIGIGN 0000000000000000' "$work/l06.in" + result "L-06 the command ignores no signal (SigIgn)" $? "($(grep '^SIGIGN' "$work/l06.in"))" + grep -qx 'SIGBLK 0000000000000000' "$work/l06.in" + result "L-06 the command blocks no signal (SigBlk)" $? "($(grep '^SIGBLK' "$work/l06.in"))" + [ "$(res "$work/l06.in" vsock -)" = EACCES ] + result "L-06 inside: an AF_VSOCK socket is refused with EACCES" $? "($(res "$work/l06.in" vsock -))" + unshare -U true 2>/dev/null + result "L-06 control: outside, unshare -U true succeeds" $? + aa-exec -p unconfined -- true 2>/dev/null + result "L-06 control: outside, aa-exec -p unconfined true succeeds" $? + python3 -I "$ro/lclient.py" vsock - >"$work/l06.out" 2>&1 + echo "INFO L-06 outside AF_VSOCK: $(res "$work/l06.out" vsock -)" + [ -n "$(res "$work/l06.out" vsock -)" ] && [ "$(res "$work/l06.out" vsock -)" != EACCES ] + result "L-06 control: outside, an AF_VSOCK socket is not refused with EACCES" $? + + # --- L-18 (with L-17) the command's output through the relays, read by + # a pipe consumer: complete, with its last stdout line a workflow command + # that must stay ignored (the resume token comes after it). + # shellcheck disable=SC2016 + l18_py=' +import sys +out = b"".join(b"%07d" % i + b"." * 120 + b"\n" for i in range(8000)) + b"::error::injected-by-test\n" +err = b"e" * 65535 + b"\n" +if sys.argv[1:] == ["sums"]: + import hashlib + print(hashlib.sha256(out).hexdigest(), hashlib.sha256(err).hexdigest()) +else: + sys.stdout.buffer.write(out) + sys.stdout.flush() + sys.stderr.buffer.write(err) +' + ndl python3 -I -c "$l18_py" 2>"$work/l18.err" | cat >"$work/l18.out" + l18_rc=${PIPESTATUS[0]} + l18_out_sum=$(python3 -I -c "$l18_py" sums | cut -d' ' -f1) + l18_token=$(sed -n 's/^::stop-commands::\([0-9a-f]*\)$/\1/p' "$work/l18.out" | head -n 1) + python3 -I -c ' +import hashlib, sys +lines = open(sys.argv[1], "rb").read().split(b"\n") +tok = sys.argv[2].encode() +try: + a = lines.index(b"::stop-commands::" + tok) + b = lines.index(b"::" + tok + b"::") +except ValueError: + sys.exit(1) +# The launcher puts the resume token on a line of its own: one empty line +# precedes it. +if b < 1 or lines[b - 1] != b"": + sys.exit(1) +body = b"\n".join(lines[a + 1:b - 1]) + b"\n" +sys.exit(0 if hashlib.sha256(body).hexdigest() == sys.argv[3] else 1) +' "$work/l18.out" "$l18_token" "$l18_out_sum" + l18_body=$? + [ "$l18_rc" -eq 0 ] && [ "$l18_body" -eq 0 ] + result "L-18 the command's 1 MiB of stdout arrives complete through a pipe" $? "(rc $l18_rc)" + l18_e=$(l18_lines "$work/l18.err") + [ "$l18_e" = 1 ] + result "L-18 the command's 64 KiB of stderr arrives complete" $? + [ -n "$l18_token" ] && [ "$(tail -n 1 "$work/l18.out")" = "::$l18_token::" ] + result "L-18 the resume token is the last stdout line" $? "($(tail -n 1 "$work/l18.out" | cut -c1-40))" + l17_err=$(grep -n '^::error::injected-by-test$' "$work/l18.out" | cut -d: -f1) + l17_res=$(grep -n "^::$l18_token::\$" "$work/l18.out" | cut -d: -f1) + [ -n "$l17_err" ] && [ -n "$l17_res" ] && [ "$l17_err" -lt "$l17_res" ] + result "L-17 the command's last workflow command comes before the resume token (ignored)" $? \ + "(line ${l17_err:-?}, resume ${l17_res:-?})" + + # --- L-14 (with L-11) a normal launch, its stdout a regular file; while + # it runs, the profile is listed and root reads the command's pending + # exec label. + g=$(new_work) || exit 2 + l14_before_ctl=$(ctl_dirs) + count_launch + "$nd" --work "$g" /bin/sh -c 'echo "$NO_DOCKER_PROFILE" >"$1/name"; i=0 +while [ ! -e "$1/go" ] && [ $i -lt 300 ]; do sleep 0.1; i=$((i + 1)); done; echo l14-out' "l14-$nonce" "$g" \ + >"$work/l14.out" 2>"$work/l14.err" & + l14pid=$! + for _ in $(seq 1 300); do [ -s "$g/name" ] && break; sleep 0.1; done + l14_name=$(cat "$g/name" 2>/dev/null) + l14_list=$(profiles 2>/dev/null | grep -c "^$l14_name (enforce)\$") + l14_cmd="" + for p in $(pgrep -f -- "l14-$nonce"); do + if [ "$(sudo -n cat "/proc/$p/attr/apparmor/current" 2>/dev/null)" = "$l14_name (enforce)" ]; then l14_cmd=$p; fi + done + l14_exec="" l14_exec_rc=1 + if [ -n "$l14_cmd" ]; then + l14_exec=$(sudo -n cat "/proc/$l14_cmd/attr/apparmor/exec" 2>&1) + l14_exec_rc=$? + fi + : >"$g/go" + wait_bounded "$l14pid" 150 + l14_rc=$? + echo "INFO L-11 root's view of the command's attr/apparmor/exec: ${l14_exec:-unread}" + grep -q '^no-docker-aa: info parser ' "$work/l14.err" && grep -q '^no-docker-aa: info features ' "$work/l14.err" + result "L-11 the preflight recorded the parser and the kernel's network features" $? + grep -q '^no-docker-aa: info apparmor_restrict_unprivileged_unconfined ' "$work/l14.err" && + grep -q '^no-docker-aa: info core_pattern ' "$work/l14.err" + result "L-11 the preflight recorded the AppArmor sysctl values and core_pattern" $? + grep -qi '^no-docker-aa: info sudo[ ].*environment' "$work/l14.err" && grep -qi '^no-docker-aa: info sudo[ ].*descriptors' "$work/l14.err" + result "L-11 the preflight recorded sudo's env_reset and closefrom" $? + [ "$l14_list" = 1 ] + result "L-11 while the command runs its profile is listed as exactly \"NAME (enforce)\"" $? "(${l14_name:-?}: $l14_list)" + grep -q '^no-docker: info the gate passed in ' "$work/l14.err" + result "L-11 the gate passed" $? + # Information only (design 7.3): the kernel answers attr/apparmor/exec + # from the reader's own context, so its content or error is recorded. + [ -n "$l14_cmd" ] + result "L-11 root's view of the command's exec label is recorded (information)" $? "(rc $l14_exec_rc)" + [ "$l14_rc" -eq 0 ] && [ "$(grep -cx l14-out "$work/l14.out")" -eq 1 ] && [ "$(listed "$l14_name")" = 0 ] + result "L-14 the command ran and its profile was removed" $? "(rc $l14_rc)" + [ "$(ctl_dirs)" = "$l14_before_ctl" ] && [ -z "$(find "$RUNNER_TEMP" -maxdepth 1 -name 'no-docker-fence.*')" ] + result "L-14 no control or fence directory is left" $? + ! pgrep -f -- 'no-docker-fence\.' >/dev/null + result "L-14 no fence listener is left" $? + grep -qE '^no-docker: info descriptors of no-docker.sh before the start: 0 1 2( [0-9]+)+; relays: stdout pid [0-9]+ on fd [0-9]+, stderr pid [0-9]+ on fd [0-9]+, both children of [0-9]+$' "$work/l14.err" + result "L-14 the launcher's descriptors and the relays' pids, fds and parent are logged" $? \ + "($(grep '^no-docker: info descriptors' "$work/l14.err" | cut -c1-120))" + l14_relays=$(sed -n 's/^no-docker: info the relays ended \([0-9]*\)\.\([0-9]*\)s after the command$/\1\2/p' "$work/l14.err") + [ -n "$l14_relays" ] && [ "$l14_relays" -le 100 ] + result "L-14 the relays ended within 1 s of the command" $? "(${l14_relays:-?} hundredths)" + ! grep -qE '^no-docker: (refused|.*still loaded|the control directory .* is kept)|^no-docker-aa: (killed pid 1|pid 1 of the launch not found|refused)' "$work/l14.err" + result "L-14 no cleanup message and no kill-init in the log of a normal end" $? + + # --- L-07 the caller's stdout a socket or a terminal: refused before + # anything is made. Controls: a pipe (L-18) and a file (L-14). + for kind in socketpair pty; do + c0=$(ctl_dirs) + p0=$(listed flowcrm-nd-) + python3 -I -c "$l07_py" "$kind" "$nd" 2>"$work/l07-$kind.err" + r=$? + [ "$r" -eq 125 ] && grep -q "the caller's fd 1 is a socket or a terminal" "$work/l07-$kind.err" && + [ "$(ctl_dirs)" = "$c0" ] && [ "$(listed flowcrm-nd-)" = "$p0" ] + result "L-07 the caller's stdout a $kind: refused (125) before anything was made" $? "(rc $r)" + done + [ "$l18_rc" -eq 0 ] + result "L-07 control: the caller's stdout a pipe runs (L-18)" $? + [ "$l14_rc" -eq 0 ] + result "L-07 control: the caller's stdout a regular file runs (L-14)" $? + + # --- L-08 the gate run directly, outside, with an extra fd 5 and fd 1 a + # socket: every step is evaluated, its report names both and the label, + # and the command does not run. The G1 findings outside are expected. + l08_py=' +import os, socket, sys +a, b = socket.socketpair() +if sys.argv[1] == "dirty": + fd = os.open("/etc/hostname", os.O_RDONLY) + if fd != 5: + os.dup2(fd, 5) + os.set_inheritable(5, True) + os.dup2(a.fileno(), 1, inheritable=True) +os.execv("/usr/bin/python3", ["/usr/bin/python3", "-I", sys.argv[2], "--gate-exec", "--", "/usr/bin/touch", sys.argv[3]]) +' + python3 -I -c "$l08_py" dirty "$root/scripts/ci/no-docker-probe.py" "$work/l08.marker" "$work/l08.err" + r=$? + [ "$r" -eq 125 ] && [ ! -e "$work/l08.marker" ] + result "L-08 the gate run with a stray descriptor refuses (125); the command did not run" $? "(rc $r)" + grep -q '^no-docker: FAIL no other descriptor is open (.*5: file' "$work/l08.err" && + grep -q '^no-docker: FAIL fd 0 is /dev/null and fds 1 and 2 are pipes (.*1: unix socket' "$work/l08.err" + result "L-08 the gate's report names fd 5 and the socket on fd 1" $? + grep -q '^no-docker: FAIL this process is confined by exactly ' "$work/l08.err" + result "L-08 the gate's report names the missing confinement" $? + { python3 -I -c "$l08_py" clean "$root/scripts/ci/no-docker-probe.py" "$work/l08.marker" &1; } | + cat >"$work/l08c.err" + grep -q '^no-docker: ok fd 0 is /dev/null and fds 1 and 2 are pipes' "$work/l08c.err" && + grep -q '^no-docker: ok no other descriptor is open$' "$work/l08c.err" && [ ! -e "$work/l08.marker" ] + result "L-08 control: clean descriptors give no descriptor finding" $? + + # --- L-09 --root-exec run directly as this user with fds 5 and 6. + l09_py=' +import os, sys +for n in map(int, sys.argv[3:]): + fd = os.open("/etc/hostname", os.O_RDONLY) + if fd != n: + os.dup2(fd, n) + os.close(fd) + os.set_inheritable(n, True) +os.execv("/usr/bin/python3", ["/usr/bin/python3", "-I", sys.argv[1], "--root-exec", str(os.getuid()), str(os.getgid()), + "flowcrm-nd-" + "0" * 32, "--", "PATH=/usr/bin:/bin", "--", "/usr/bin/touch", sys.argv[2]]) +' + python3 -I -c "$l09_py" "$root/scripts/ci/no-docker-probe.py" "$work/l09.marker" 5 6 /dev/null 2>"$work/l09.err" + r=$? + grep -q '^no-docker: closed: 5 6; remaining: 0 1 2; stdin char 1:3$' "$work/l09.err" + result "L-09 --root-exec closed the inherited fds 5 and 6 and made stdin /dev/null" $? "($(grep closed: "$work/l09.err"))" + [ "$r" -ne 0 ] && [ ! -e "$work/l09.marker" ] && grep -q '^setpriv: ' "$work/l09.err" + result "L-09 then setpriv failed without privileges; the command did not run" $? "(rc $r)" + python3 -I -c "$l09_py" "$root/scripts/ci/no-docker-probe.py" "$work/l09.marker" /dev/null 2>"$work/l09b.err" + grep -q '^no-docker: closed: (none); remaining: 0 1 2; stdin char 1:3$' "$work/l09b.err" + result "L-09 control: without them nothing is closed" $? + + # --- L-10 a host process that would pass a descriptor (SCM_RIGHTS). + mkdir -m 700 "$ro/l10" || exit 2 + printf 'l10 secret\n' >"$ro/l10/secret" + lsock scm "$ro/l10/s" "$work/l10.ready" "$ro/l10/secret" + python3 -I "$ro/lclient.py" scm "$ro/l10/s" >"$work/l10.out" 2>&1 + ndl python3 -I "$ro/lclient.py" scm "$ro/l10/s" fds - >"$work/l10.in" 2>"$work/l10.err" + [ "$(res "$work/l10.in" scm "$ro/l10/s")" = EACCES ] + result "L-10 inside: the connect to the descriptor-passing socket is refused with EACCES" $? "($(res "$work/l10.in" scm "$ro/l10/s"))" + [ "$(res "$work/l10.out" scm "$ro/l10/s")" = "OK fds=1" ] && [ "$(res "$work/l10.in" scm "$ro/l10/s")" != "OK fds=1" ] + result "L-10 no descriptor reached the command (outside, a client receives one)" $? + [ "$(res "$work/l10.in" fds -)" = "0 1 2" ] + result "L-10 the command's descriptors are 0, 1 and 2" $? "($(res "$work/l10.in" fds -))" + lstop "$work/l10.ready" + + # --- L-12 the negative control: a profile that does not deny host + # sockets must make the gate refuse; only /usr/bin/true may run. + ndl --negative-control /usr/bin/touch "$work/l12.marker" >"$work/l12.out" 2>"$work/l12.err" + r=$? + [ "$r" -eq 125 ] + result "L-12 the negative control exits 125" $? "(rc $r)" + grep -q '^no-docker: FAIL connect to the stream fence is refused with EACCES (ok)$' "$work/l12.err" && + grep -q '^no-docker: FAIL sendto to the dgram fence is refused with EACCES (ok)$' "$work/l12.err" && + grep -q '^no-docker: ok connect to the stream fence is refused with EACCES (EACCES)$' "$work/l14.err" + result "L-12 the gate reports the fence reachable (stream and datagram); L-14's normal launch found it refused" $? + [ ! -e "$work/l12.marker" ] && ! grep -q 'no-docker: info the gate passed' "$work/l12.err" + result "L-12 only /usr/bin/true could have run (the given command did not)" $? + [ "$(listed flowcrm-nd-neg-)" = 0 ] && grep -q '^no-docker-aa: released flowcrm-nd-neg-' "$work/l12.err" + result "L-12 the negative-control profile was released" $? + + # --- L-13 the helper keeps a profile a process outside still carries. + l13n=flowcrm-nd-$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') + l13c=$(mktemp -d "$RUNNER_TEMP/no-docker-ctl.XXXXXX") || exit 2 + printf 'loading\n' >"$l13c/state" + printf '%s %s\n' "$l13n" "$l13c" >>"$work/l-release" + sudo -n timeout 20 /usr/bin/python3 -I "$root/scripts/ci/no-docker-aa.py" load "$l13n" "$l13c" 2>"$work/l13.err" + aa-exec -p "$l13n" -- sleep 120 & + l13carrier=$! + printf '%s\n' "$l13carrier" >>"$work/l-carriers" + sleep 0.5 + sudo -n timeout 30 /usr/bin/python3 -I "$root/scripts/ci/no-docker-aa.py" release "$l13n" "$l13c" 2>>"$work/l13.err" + r=$? + [ "$r" -ne 0 ] && grep -q 'tasks carrying the profile' "$work/l13.err" + result "L-13 release is refused while a process outside carries the profile" $? "(rc $r)" + [ "$(listed "$l13n (enforce)")" = 1 ] + result "L-13 the profile is still loaded after the refusal" $? + kill "$l13carrier" 2>/dev/null + wait "$l13carrier" 2>/dev/null + sudo -n timeout 30 /usr/bin/python3 -I "$root/scripts/ci/no-docker-aa.py" release "$l13n" "$l13c" 2>>"$work/l13.err" + r=$? + [ "$r" -eq 0 ] && [ "$(listed "$l13n")" = 0 ] + result "L-13 once the carrier ended the release removes it" $? "(rc $r)" + rm -rf "$l13c" + + # --- L-13b the launcher's own release refused: 125, the control + # directory kept, the profile releasable afterwards. + g=$(new_work) || exit 2 + count_launch + "$nd" --work "$g" /bin/sh -c 'echo "$NO_DOCKER_PROFILE" >"$1/name"; i=0 +while [ ! -e "$1/go" ] && [ $i -lt 300 ]; do sleep 0.1; i=$((i + 1)); done' l13b "$g" >"$work/l13b.out" 2>"$work/l13b.err" & + l13bpid=$! + for _ in $(seq 1 300); do [ -s "$g/name" ] && break; sleep 0.1; done + l13b_name=$(cat "$g/name" 2>/dev/null) + case $l13b_name in flowcrm-nd-????????????????????????????????) aa-exec -p "$l13b_name" -- sleep 120 & ;; *) sleep 120 & ;; esac + l13bcarrier=$! + printf '%s\n' "$l13bcarrier" >>"$work/l-carriers" + sleep 0.5 + : >"$g/go" + wait_bounded "$l13bpid" 150 + r=$? + l13b_line=$(grep -m1 '^no-docker: refused: profile .* still loaded' "$work/l13b.err") + l13b_ctl=${l13b_line##*control directory } + l13b_ctl=${l13b_ctl%% is kept} + [ "$r" -eq 125 ] && [ -n "$l13b_line" ] + result "L-13b with a carrier outside the launch ends 125 and says the profile is still loaded" $? "(rc $r)" + [ -d "$l13b_ctl" ] && [ -e "$l13b_ctl/ns" ] + result "L-13b its control directory is kept" $? "(${l13b_ctl:-none})" + kill "$l13bcarrier" 2>/dev/null + wait "$l13bcarrier" 2>/dev/null + left_loaded "$work/l13b.err" && [ "$(listed "$l13b_name")" = 0 ] + result "L-13b without the carrier the profile is released with that control directory" $? + + # --- L-13c two launches at once: each releases only its own profile. + l13c_tmp=$(find /tmp -maxdepth 1 -name 'no-docker.*' 2>/dev/null | wc -l | tr -d ' ') + count_launch + "$nd" /bin/sh -c 'sleep 2; echo l13c-a' >"$work/l13c-a.out" 2>"$work/l13c-a.err" & + l13ca=$! + count_launch + "$nd" /bin/sh -c 'sleep 2; echo l13c-b' >"$work/l13c-b.out" 2>"$work/l13c-b.err" & + l13cb=$! + wait_bounded "$l13ca" 150 + ra=$? + wait_bounded "$l13cb" 150 + rb=$? + na=$(sed -n 's/^no-docker-aa: released \(flowcrm-nd-[0-9a-f]*\) .*/\1/p' "$work/l13c-a.err") + nb=$(sed -n 's/^no-docker-aa: released \(flowcrm-nd-[0-9a-f]*\) .*/\1/p' "$work/l13c-b.err") + [ "$ra" -eq 0 ] && [ "$rb" -eq 0 ] && grep -qx l13c-a "$work/l13c-a.out" && grep -qx l13c-b "$work/l13c-b.out" + result "L-13c two concurrent launches both run" $? "(rc $ra, $rb)" + [ -n "$na" ] && [ -n "$nb" ] && [ "$na" != "$nb" ] && [ "$(grep -c '^no-docker-aa: released' "$work/l13c-a.err")" = 1 ] && + [ "$(grep -c '^no-docker-aa: released' "$work/l13c-b.err")" = 1 ] + result "L-13c each released its own profile, once" $? + [ "$(listed flowcrm-nd-)" = "$nd_listed_before" ] && [ "$(ctl_dirs)" = 0 ] && + [ "$(find /tmp -maxdepth 1 -name 'no-docker.*' 2>/dev/null | wc -l | tr -d ' ')" = "$l13c_tmp" ] + result "L-13c no profile, control directory or run directory of theirs is left" $? + + # --- L-15 TERM to no-docker.sh (not a runner cancellation, RR-01): 143, + # pid 1 killed, the profile released only after P1, nothing left. + g=$(new_work) || exit 2 + count_launch + "$nd" --work "$g" /bin/sh -c 'sleep 120' "l15-$nonce" >"$work/l15.out" 2>"$work/l15.err" & + l15pid=$! + for _ in $(seq 1 300); do [ -e "$g/started" ] && break; sleep 0.1; done + kill -TERM "$l15pid" + wait_bounded "$l15pid" 150 + r=$? + [ "$r" -eq 143 ] + result "L-15 TERM: no-docker.sh exits 143 within 150 s" $? "(rc $r)" + grep -q '^no-docker-aa: killed pid 1 of the launch' "$work/l15.err" + result "L-15 pid 1 of the namespace was killed (kill-init)" $? + grep -q '^no-docker-aa: released flowcrm-nd-[0-9a-f]* (namespace, state entered)$' "$work/l15.err" + result "L-15 the profile was released after P0 and P1" $? + ! pgrep -f -- "l15-$nonce" >/dev/null + result "L-15 no process of the launch is left" $? + + # --- L-15b unshare killed by this suite while the command forks and + # tries a host socket: released only after P1, or left loaded (125); the + # host socket never accepted a connection. + mkdir -m 700 "$ro/l15b" || exit 2 + lsock stream "$ro/l15b/s" "$work/l15b.ready" + g=$(new_work) || exit 2 + count_launch + "$nd" --work "$g" /bin/sh -c 'while :; do python3 -I "$1" stream "$2" >/dev/null 2>&1 & sleep 0.05; done' \ + "l15b-$nonce" "$ro/lclient.py" "$ro/l15b/s" >"$work/l15b.out" 2>"$work/l15b.err" & + l15bpid=$! + for _ in $(seq 1 300); do [ -e "$g/started" ] && break; sleep 0.1; done + sleep 1 + l15b_unshare="" + for p in $(pgrep -x unshare); do + q=$p + for _ in 1 2 3 4 5 6; do + q=$(awk '{print $4}' "/proc/$q/stat" 2>/dev/null) || break + [ "$q" = "$l15bpid" ] && l15b_unshare=$p && break + done + done + [ -z "$l15b_unshare" ] || sudo -n kill -KILL "$l15b_unshare" + wait_bounded "$l15bpid" 150 + r=$? + if [ "$r" -eq 125 ]; then + left_loaded "$work/l15b.err" + l15b_ok=$? + else + grep -q '^no-docker-aa: released flowcrm-nd-[0-9a-f]* (namespace, state entered)$' "$work/l15b.err" + l15b_ok=$? + fi + [ -n "$l15b_unshare" ] && [ "$l15b_ok" -eq 0 ] && { [ "$r" -eq 125 ] || [ "$r" -eq 137 ]; } + result "L-15b unshare killed: released after P1, or left loaded (125) and released afterwards" $? \ + "(rc $r, unshare ${l15b_unshare:-not found})" + ! grep -q '^no-docker-aa: released .*(no-namespace, state entere' "$work/l15b.err" + result "L-15b never released as if no namespace had been made" $? + [ "$(cat "$work/l15b.ready.count" 2>/dev/null || echo 0)" -eq 0 ] + result "L-15b the host socket never accepted a connection from inside" $? + lstop "$work/l15b.ready" + + # --- L-15c TERM early in the launch, at six moments. Each launch ends in + # one of: 143 with nothing made or loaded; 143 released (after P1, or as + # loading/not-forked without a namespace); 125 left loaded, released + # afterwards once its processes are gone. Never released in "entering" or + # "entered" without P1. + for ms in 0 20 50 100 200 300; do + count_launch + "$nd" /usr/bin/true >"$work/l15c-$ms.out" 2>"$work/l15c-$ms.err" & + l15cpid=$! + sleep "$(printf '0.%03d' "$ms")" + kill -TERM "$l15cpid" 2>/dev/null + wait_bounded "$l15cpid" 150 + r=$? + l15c_ok=no + if [ "$r" -eq 143 ] && ! grep -q 'still loaded' "$work/l15c-$ms.err"; then + l15c_ok=yes + elif [ "$r" -eq 125 ] && left_loaded "$work/l15c-$ms.err"; then + l15c_ok=yes + fi + grep -q '^no-docker-aa: released .*(no-namespace, state entere' "$work/l15c-$ms.err" && l15c_ok=no + if [ "$ms" = 300 ]; then + [ "$(listed flowcrm-nd-)" = "$nd_listed_before" ] || l15c_ok=no + fi + l15c_result "$ms" "$l15c_ok" "$r" + done +else + echo "INFO T-16, T-17, T-18 skipped: Linux (GitHub-hosted runner) only" + for c in L-01 L-02 L-03 L-04 L-05 L-06 L-07 L-08 L-09 L-10 L-11 L-12 L-13 L-13b L-13c L-14 L-15 L-15b L-15c L-17 L-18; do + echo "SKIP $c (Linux CI only)" + done + # --- T-18 (macOS) the negative control is refused off Linux. + err=$("$nd" --negative-control /usr/bin/true 2>&1 >/dev/null) + r=$? + [ "$r" -eq 125 ] && printf '%s' "$err" | grep -q -- "--negative-control is for Linux on GitHub-hosted runners only" + result "T-18 --negative-control on macOS: refused" $? "(rc $r)" +fi + +# --- T-20 code that runs inside the isolation uses no here-document or +# here-string (static): on macOS bash 3.2 would need /tmp for them, which +# the sandbox does not allow, so one would fail there (and a missed failure +# can look like a pass). The suites, the scripts they run and bin/devenv; +# the payload is checked by its hash. The one allowed match is the edge +# guard's test text for a Caddyfile here-document. +t20=$(cd "$root" && grep -nE "(^|[^<])<<-?[[:space:]]*[\"'\\\\]?[A-Za-z_]|<<<" -- bin/devenv tests/devenv/unit.sh \ + tests/devenv/tally.sh scripts/ci/no-docker.sh scripts/ci/guard-*.sh scripts/ci/tests/guard-*.test.sh 2>&1 | + grep -vE '^[^:]+:[0-9]+:[[:space:]]*#' | grep -vF 'heredoc (<<) is not supported') +printf '%s\n' "$t20" | grep . | sed 's/^/INFO /' +[ -z "$t20" ] +result "T-20 code run inside the isolation has no here-document or here-string" $? + +# --- T-23 the helpers this suite runs outside any isolation are pinned to +# reviewed hashes in this file (static): none is hashed after it was +# written. On dfb6ee4: canary.py and drive.sh were written with +# here-documents and their sha256 taken afterwards. +grep -qE '^helper_sums="canary\.py=[0-9a-f]{64} drive\.sh=[0-9a-f]{64} payload=[0-9a-f]{64}"$' "$here/no-docker.test.sh" && + ! grep -nE '^[^#]*helper_sums=.*\$\(' "$here/no-docker.test.sh" >/dev/null && + ! grep -nE '^[^#]*>"?\$helpers/' "$here/no-docker.test.sh" >/dev/null +result "T-23 the helpers run outside any isolation are pinned to reviewed sha256 values, not hashed after writing" $? + +# --- T-36 the Linux rules of no-docker-probe.py as pure functions, on both +# platforms: fixtures only (every call that would read /proc, open, close or +# duplicate a descriptor, open a socket, run or exec a program, fork or set +# a signal raises), the gate's label, descriptor and fence rules and +# --root-exec's sanitation (C11 to C14), the probe's functions called by +# what runs (W) and mutated copies of the probe rejected (M1 to M21). The +# numbers of assertions are fixed here: a case that loses one fails. +# On 85cfb37: the NSpid rule accepted only a /proc of an ancestor namespace +# (M1). +python3 -I -B "$here/no-docker-probe-unit.py" "$root/scripts/ci/no-docker-probe.py" >"$work/T-36.out" 2>&1 +t36_rc=$? +t36=$(cat "$work/T-36.out") +printf '%s\n' "$t36" | grep -vE '^ok ' | sed 's/^/INFO /' +for want in C01=2 C02=13 C03=12 C04=14 C05=9 C06=15 C07=7 C10=8 C11=7 C12=9 C13=9 C14=4 W=12 M=14; do + tag=${want%=*} + n=$(printf '%s\n' "$t36" | grep -cE "^ok ${tag}[0-9]* ") + bad=$(printf '%s\n' "$t36" | grep -cE "^FAIL ${tag}[0-9]* ") + [ "$t36_rc" -eq 0 ] && [ "$n" -eq "${want#*=}" ] && [ "$bad" -eq 0 ] && + printf '%s\n' "$t36" | grep -qx 'SUMMARY 135 ok, 0 failed' + result "T-36 $tag: $n of ${want#*=} assertions pass" $? "(rc $t36_rc, $bad failed)" +done + +# --- T-39 the static rules of the Linux isolation (no-docker-static.py), on +# both platforms: each rule passes on the tracked files and fails on its +# mutant fixture; the AppArmor templates are the reviewed rule set. The +# numbers of checks are fixed here: a rule that loses one fails. +python3 -I -B "$here/no-docker-static.py" "$root" >"$work/T-39.out" 2>&1 +t39_rc=$? +t39=$(cat "$work/T-39.out") +printf '%s\n' "$t39" | grep -vE '^ok ' | sed 's/^/INFO /' +for want in S-02=2 S-02b=2 S-03=2 S-03b=2 S-04=2 S-05=4 S-06=2 S-07=2 S-08=2 S-09=2; do + tag=${want%=*} + n=$(printf '%s\n' "$t39" | grep -cE "^ok ${tag} ") + bad=$(printf '%s\n' "$t39" | grep -cE "^FAIL ${tag} ") + [ "$t39_rc" -eq 0 ] && [ "$n" -eq "${want#*=}" ] && [ "$bad" -eq 0 ] && + printf '%s\n' "$t39" | grep -qx 'SUMMARY 22 ok, 0 failed' + result "T-39 $tag: $n of ${want#*=} checks pass" $? "(rc $t39_rc, $bad failed)" +done + +# --- T-40 the Linux cleanup's failure propagation, as a pure harness, on +# both platforms: the cleanup function of no-docker.sh and the functions it +# calls, taken from the script unchanged, run in a subshell in the state of +# a launch whose command exited 0, with rm, sudo and chmod replaced by shell +# functions (only this suite's own directories are removed; nothing runs as +# root; no namespace or profile exists). A failed removal of the fence, the +# control or the run directory must end 125 with a refusal naming that +# directory by its path, and leave the directories the design keeps (design +# section 10, C3, C6, C7): after a fence failure the control directory is +# kept too; after a control or run failure what was removed before stays +# removed. Without a failure the command's status stays 0 and nothing is +# left. Four mutants of the propagation must each change an outcome. The +# shell logic only: on Linux, L-14 and L-13c show at run time that nothing +# is left. +t40_src=$work/t40-cleanup.sh +awk ' + /^ (proc_stat|alive|wait_gone|wait_all|put_state|resolve_pending|sudo_may_run|close_relay_ends|secs|cleanup)\(\) \{$/ { f = 1 } + f { print } + f && /^ }$/ { f = 0 } +' "$nd" >"$t40_src" +# The harness, a shell of its own: $1 the functions, $2 the case's +# directory, $3 the directory whose removal fails. +t40_harness=' +set -u +EX_REFUSED=125 aa=/nonexistent EPOCHREALTIME=1700000000.000000 bad=$3 +ctldir=$2/ctl fencedir=$2/fence work=$2/run work_made=yes profile=flowcrm-nd-00000000000000000000000000000000 +load_op=load state="entered spid=1" lpid=1 lstart=1 load_done=yes spid=1 sstart=1 sudo_reaped=yes +o="" e="" opid="" ostart="" epid="" estart="" fspid="" fsstart="" fdpid="" fdstart="" +pending="" kept="" token="" stopped="" signalled="" normal=yes cleaning="" rc=0 negative="" +t_launch=0 t_preflight=0 t_load=0 t_fence=0 t_relays=0 t_run=0 t_release=0 +sudo() { :; } +chmod() { :; } +rm() { + for last; do :; done + [ "$last" != "$bad" ] || return 1 + command rm "$@" +} +. "$1" +cleanup +' +# t40_case SOURCE fence|ctl|run|none: "::". +t40_case() { + local d r x name left="" + d=$(mktemp -d "$work/t40.XXXXXX") || return 2 + mkdir "$d/fence" "$d/ctl" "$d/run" && : >"$d/run/started" || return 2 + "$BASH" -c "$t40_harness" t40 "$1" "$d" "$d/$2" >"$d/out" 2>"$d/err" + r=$? + case $2 in fence) name=fence ;; ctl) name=control ;; *) name=run ;; esac + for x in fence ctl run; do + if [ -e "$d/$x" ]; then left=$left$(printf %.1s "$x"); else left=$left-; fi + done + echo "$r:$(grep -cxF "no-docker: refused: cleanup failed (the $name directory $d/$2 could not be removed)" "$d/err"):$left" +} +t40_all() { # SOURCE: the outcomes of the four cases, space-separated + echo "none=$(t40_case "$1" none) fence=$(t40_case "$1" fence) ctl=$(t40_case "$1" ctl) run=$(t40_case "$1" run)" +} +t40_want="none=0:0:--- fence=125:1:fc- ctl=125:1:-c- run=125:1:--r" +t40_got=$(t40_all "$t40_src") +[ "$(grep -c '() {$' "$t40_src")" -eq 10 ] && [ "$t40_got" = "$t40_want" ] +result "T-40 cleanup against stubs: a failed removal of the fence, control or run directory ends 125 naming it and keeps what the design keeps; otherwise status 0, nothing left" $? \ + "($t40_got; $(grep -c '() {$' "$t40_src") of 10 functions)" +# The mutants: the fence removal unchecked; the failure cleared before the +# status; the status not raised by a cleanup failure; the fence failure +# naming another directory. +awk '/^ if \[ -n "\$fencedir" \] && ! rm -rf "\$fencedir"; then$/ { print " [ -z \"$fencedir\" ] || rm -rf \"$fencedir\""; skip = 2; next } + skip > 0 { skip--; next } { print }' "$t40_src" >"$work/t40-m1.sh" +awk '/^ # C7: the status\./ { print " fail=\"\"" } { print }' "$t40_src" >"$work/t40-m2.sh" +awk '/^ # C7: the status\./ { c7 = 1 } c7 && /^ status=\$EX_REFUSED$/ && !done { print " status=$rc"; done = 1; next } { print }' \ + "$t40_src" >"$work/t40-m3.sh" +awk '/the fence directory \$fencedir could not be removed/ { sub(/\$fencedir could/, "$ctldir could") } { print }' \ + "$t40_src" >"$work/t40-m4.sh" +t40_caught="" +for m in 1 2 3 4; do + if ! cmp -s "$t40_src" "$work/t40-m$m.sh" && [ "$(t40_all "$work/t40-m$m.sh")" != "$t40_want" ]; then + t40_caught="$t40_caught M$m" + fi +done +[ "$t40_caught" = " M1 M2 M3 M4" ] +result "T-40 mutants of the propagation are caught (fence removal unchecked, failure cleared, status not raised, another directory named)" $? \ + "(caught:${t40_caught:- none})" + +# --- T-41 L-16's projection is compared with 540 s exactly (pure, both +# platforms): l16_project, which L-16 uses, sums the launch timings (two +# decimals, as the launch notices print them) in integer hundredths, so a +# projection of 540.04 s is over the limit although it would print as 540. +# Cases: exactly 540 s passes (N = 40, ten launches of 8.90 s); 540.004 s +# fails (N = 4, nine of 89.00 s and one of 89.01 s); 540.04 s fails (N = 45, +# nine of 7.91 s and one of 7.93 s); 539.95 s passes (N = 45, ten of 7.91 s); +# a malformed timing is refused. The printed c and projection are rounded +# up. +# l16_project N TIMING...: " ", or "- - bad"; the projection is 184 s +# plus N times c, c the mean of the timings. +l16_project() { + local n=$1 k=0 s=0 t c p + shift + for t in "$@"; do + case $t in + "" | *[!0-9.]* | *.*.*) echo "- - bad"; return 1 ;; + [0-9]*.[0-9][0-9]) s=$((s + 10#${t%.*} * 100 + 10#${t#*.})) k=$((k + 1)) ;; + *) echo "- - bad"; return 1 ;; + esac + done + case $n in "" | *[!0-9]*) echo "- - bad"; return 1 ;; esac + [ "$k" -gt 0 ] || { echo "- - bad"; return 1; } + # c = s / (100 k) s; 100 k times the projection = 18400 k + n s. + c=$(((10 * s + k - 1) / k)) + p=$(((18400 * k + n * s + 100 * k - 1) / (100 * k))) + if [ $((18400 * k + n * s)) -le $((54000 * k)) ]; then t=ok; else t=over; fi + printf '%d.%03d %d %s\n' $((c / 1000)) $((c % 1000)) "$p" "$t" +} +t41_got="$(l16_project 40 8.90 8.90 8.90 8.90 8.90 8.90 8.90 8.90 8.90 8.90)|$(l16_project 4 89.00 89.00 89.00 89.00 89.00 89.00 89.00 89.00 89.00 89.01)|$(l16_project 45 7.91 7.91 7.91 7.91 7.91 7.91 7.91 7.91 7.91 7.93)|$(l16_project 45 7.91 7.91 7.91 7.91 7.91 7.91 7.91 7.91 7.91 7.91)|$(l16_project 45 7.9 8.00)" +[ "$t41_got" = "8.900 540 ok|89.001 541 over|7.912 541 over|7.910 540 ok|- - bad" ] && + grep -q '^ l16=$(l16_project "$l16_launches" $l16_t)$' "$here/no-docker.test.sh" +result "T-41 L-16's projection is compared with 540 s exactly: 540 s passes, 540.004 s and 540.04 s fail, 539.95 s passes" $? \ + "($t41_got)" + +# --- T-42 the opt-in copy of the case outputs (NO_DOCKER_TEST_EVIDENCE), +# pure, both platforms, on directories of this suite: regular files are +# copied with their paths and sha256; a symbolic link, a FIFO and a file +# with an unsafe name are skipped and recorded; a destination that is not +# empty or not mode 0700 is refused (2). Then no byte from outside the +# copied tree is kept: a directory replaced by a link to outside between +# its listing and the descent, or after it was opened, a link to a +# directory, a destination with a link among its ancestors (refused), a +# file with a second hard link, and a source given by a link or with a +# link among its ancestors (refused). On 51013e5 the four cases after the +# first fail: the copy walked by path and kept the outside files; on +# 9ac29bb the two last fail: the copy resolved the source path first. +t42=$work/t42 +mkdir -p "$t42/src/sub" "$t42/dest" "$t42/full" "$t42/open" && chmod 700 "$t42/dest" "$t42/full" && chmod 755 "$t42/open" && + printf 'a\n' >"$t42/src/a.log" && printf 'b\n' >"$t42/src/sub/b.log" && ln -s /etc/hosts "$t42/src/link" && + mkfifo "$t42/src/fifo" && printf 'x\n' >"$t42/src/bad name" && : >"$t42/full/x" || exit 2 +python3 -I -c "$keep_py" copy "$t42/dest" "work=$t42/src" >/dev/null 2>&1 +t42_rc=$? +python3 -I -c "$keep_py" check "$t42/full" >/dev/null 2>&1 +t42_full=$? +python3 -I -c "$keep_py" check "$t42/open" >/dev/null 2>&1 +t42_open=$? +t42_want="$(sha "$t42/src/a.log") work/a.log +$(sha "$t42/src/sub/b.log") work/sub/b.log" +[ "$t42_rc" -eq 0 ] && [ "$(cat "$t42/dest/MANIFEST.sha256" 2>/dev/null)" = "$t42_want" ] && + cmp -s "$t42/src/a.log" "$t42/dest/work/a.log" && cmp -s "$t42/src/sub/b.log" "$t42/dest/work/sub/b.log" && + [ ! -e "$t42/dest/work/link" ] && [ ! -e "$t42/dest/work/fifo" ] && + [ "$(grep -c -e 'work/link' -e 'work/fifo' -e "work/bad name" "$t42/dest/SKIPPED.txt" 2>/dev/null)" -eq 3 ] && + [ "$t42_full" -eq 2 ] && [ "$t42_open" -eq 2 ] +result "T-42 the opt-in copy of the case outputs keeps regular files with sha256, skips links, FIFOs and unsafe names, refuses an unsuitable destination" $? \ + "(rc $t42_rc, full $t42_full, open $t42_open)" +# The races: the copy runs through t42_race_py, which wraps the os functions +# CALLS and, before the first of those calls whose path ends in NAME, +# renames the directory TARGET aside and puts a symbolic link to $t42/out +# (files of the same names, holding OUTSIDE-42) in its place: the +# replacement happens exactly in the window under test. It exits 3 if the +# replacement never happened, else with the copy's status. +t42_race_py=' +import os, sys +calls, name, target, outside, code = sys.argv[1:6] +done = [] +def wrap(real): + def call(*a, **k): + if not done and a and isinstance(a[0], str) and os.path.basename(a[0]) == name: + done.append(a[0]) + os.rename(target, target + ".moved") + os.symlink(outside, target) + return real(*a, **k) + return call +for f in calls.split(","): + setattr(os, f, wrap(getattr(os, f))) +sys.argv = ["-c"] + sys.argv[6:] +rc = 0 +try: + exec(compile(code, "", "exec"), {"__name__": "__main__"}) +except SystemExit as e: + rc = e.code if isinstance(e.code, int) else 1 +sys.exit(rc if done else 3) +' +mkdir -p "$t42/out" "$t42/r1/src/d42" "$t42/r1/dest" "$t42/r2/src/d42" "$t42/r2/dest" "$t42/r3/real/dest" "$t42/r4/src" "$t42/r4/dest" && + chmod 700 "$t42/r1/dest" "$t42/r2/dest" "$t42/r3/real/dest" "$t42/r4/dest" && + printf 'OUTSIDE-42\n' >"$t42/out/f42.log" && printf 'OUTSIDE-42 extra\n' >"$t42/out/extra.log" && + printf 'inside\n' >"$t42/r1/src/d42/f42.log" && printf 'top\n' >"$t42/r1/src/a.log" && ln -s "$t42/out" "$t42/r1/src/dlink" && + printf 'inside\n' >"$t42/r2/src/d42/f42.log" && ln -s real "$t42/r3/alias" && + printf 'top\n' >"$t42/r4/src/a.log" && ln "$t42/out/f42.log" "$t42/r4/src/hard.log" || exit 2 +# A directory replaced after the listing, before the descent (the first +# open or listing of it), and a link to a directory: not followed. +python3 -I -c "$t42_race_py" open,scandir,listdir d42 "$t42/r1/src/d42" "$t42/out" "$keep_py" copy "$t42/r1/dest" "work=$t42/r1/src" >/dev/null 2>&1 +t42_r1=$? +[ "$t42_r1" -eq 0 ] && ! grep -rq OUTSIDE-42 "$t42/r1/dest" && [ ! -e "$t42/r1/dest/work/d42" ] && [ ! -e "$t42/r1/dest/work/dlink" ] && + cmp -s "$t42/r1/src/a.log" "$t42/r1/dest/work/a.log" && grep -q '^work/d42: E[A-Z]* (directory replaced' "$t42/r1/dest/SKIPPED.txt" && + grep -q '^work/dlink: symbolic link, not followed$' "$t42/r1/dest/SKIPPED.txt" +result "T-42 a directory replaced by a link to outside between its listing and the descent, and a link to a directory, are not followed: no byte from outside is kept" $? \ + "(rc $t42_r1)" +# A directory replaced after it was opened, before its file is read (the +# first stat or open of the file): read through the directory opened. +python3 -I -c "$t42_race_py" open,stat,lstat f42.log "$t42/r2/src/d42" "$t42/out" "$keep_py" copy "$t42/r2/dest" "work=$t42/r2/src" >/dev/null 2>&1 +t42_r2=$? +[ "$t42_r2" -eq 0 ] && ! grep -rq OUTSIDE-42 "$t42/r2/dest" && [ "$(cat "$t42/r2/dest/work/d42/f42.log" 2>/dev/null)" = inside ] && + [ "$(cat "$t42/r2/dest/MANIFEST.sha256" 2>/dev/null)" = "$(sha "$t42/r2/src/d42.moved/f42.log") work/d42/f42.log" ] +result "T-42 a directory replaced by a link to outside after it was opened: its file is read through the directory opened, no byte from outside is kept" $? \ + "(rc $t42_r2)" +# A destination with a symbolic link among its ancestors. +python3 -I -c "$keep_py" check "$t42/r3/alias/dest" >/dev/null 2>&1 +t42_r3c=$? +python3 -I -c "$keep_py" copy "$t42/r3/alias/dest" "work=$t42/r4/src" >/dev/null 2>&1 +t42_r3=$? +[ "$t42_r3c" -eq 2 ] && [ "$t42_r3" -eq 2 ] && [ -z "$(ls -A "$t42/r3/real/dest")" ] +result "T-42 a destination with a symbolic link among its ancestors is refused (2) and nothing is written" $? \ + "(check $t42_r3c, copy $t42_r3)" +# A file with a second hard link (here to a file outside). +python3 -I -c "$keep_py" copy "$t42/r4/dest" "work=$t42/r4/src" >/dev/null 2>&1 +t42_r4=$? +[ "$t42_r4" -eq 0 ] && ! grep -rq OUTSIDE-42 "$t42/r4/dest" && [ ! -e "$t42/r4/dest/work/hard.log" ] && + cmp -s "$t42/r4/src/a.log" "$t42/r4/dest/work/a.log" && grep -q '^work/hard.log: 2 hard links$' "$t42/r4/dest/SKIPPED.txt" +result "T-42 a file with a second hard link is not copied (skipped and recorded)" $? \ + "(rc $t42_r4)" +# Sources given through a symbolic link to outside: the root itself, and an +# ancestor of it. +mkdir -p "$t42/r5/out/src" "$t42/r5/dest1" "$t42/r5/dest2" && chmod 700 "$t42/r5/dest1" "$t42/r5/dest2" && + printf 'OUTSIDE-42\n' >"$t42/r5/out/src/f42.log" && ln -s out/src "$t42/r5/rootlink" && ln -s out "$t42/r5/alink" || exit 2 +python3 -I -c "$keep_py" copy "$t42/r5/dest1" "work=$t42/r5/rootlink" >/dev/null 2>&1 +t42_r5=$? +[ "$t42_r5" -eq 2 ] && [ -z "$(ls -A "$t42/r5/dest1")" ] +result "T-42 a source given by a symbolic link to outside is refused (2) and nothing is copied" $? \ + "(rc $t42_r5)" +python3 -I -c "$keep_py" copy "$t42/r5/dest2" "work=$t42/r5/alink/src" >/dev/null 2>&1 +t42_r6=$? +[ "$t42_r6" -eq 2 ] && [ -z "$(ls -A "$t42/r5/dest2")" ] +result "T-42 a source with a symbolic link to outside among its ancestors is refused (2) and nothing is copied" $? \ + "(rc $t42_r6)" + +# --- T-43 a stream a case reads is kept, or the suite fails (keep_stream), +# pure, both platforms: each mode keeps exactly the stream it copies and +# returns the command's own status; a copy that cannot be written ends the +# suite with 2 even after the status a case expects (125); the launches of +# T-02, T-07, T-16, T-17 and T-18 use it, and no other tee pipeline exists. +t43=$work/t43 +mkdir -p "$t43/ro" && chmod 500 "$t43/ro" || exit 2 +t43_cmd='printf "out1\nout2\n"; printf "err1\n" >&2; exit 125' +keep_stream both "$t43/both.out" /bin/sh -c "$t43_cmd" +t43_b=$ks_rc +keep_stream out "$t43/out.out" "$t43/out.err" /bin/sh -c "$t43_cmd" +t43_o=$ks_rc +keep_stream err "$t43/err.err" "$t43/err.out" /bin/sh -c "$t43_cmd" +t43_e=$ks_rc +[ "$t43_b" -eq 125 ] && [ "$t43_o" -eq 125 ] && [ "$t43_e" -eq 125 ] && + [ "$(sort "$t43/both.out" | tr '\n' ' ')" = "err1 out1 out2 " ] && + [ "$(cat "$t43/out.out")" = "$(printf 'out1\nout2')" ] && [ "$(cat "$t43/out.err")" = err1 ] && + [ "$(cat "$t43/err.err")" = err1 ] && [ "$(cat "$t43/err.out")" = "$(printf 'out1\nout2')" ] +result "T-43 keep_stream keeps exactly the stream of each mode and returns the command's own status" $? \ + "(statuses $t43_b $t43_o $t43_e)" +( + keep_stream both "$t43/ro/x.out" /bin/sh -c "$t43_cmd" + echo "continued after status $ks_rc" +) >"$t43/fail.out" 2>"$t43/fail.err" +t43_r=$? +[ "$t43_r" -eq 2 ] && ! grep -q continued "$t43/fail.out" && + grep -q '^HARNESS ERROR: the raw output .*/ro/x\.out could not be written (tee status [1-9][0-9]*, command status 125)$' "$t43/fail.err" +result "T-43 a raw output that cannot be written ends the suite with 2 (harness error), even when the command ends 125 as a case expects" $? \ + "(rc $t43_r)" +chmod 700 "$t43/ro" +t43_sites=$(grep -cE '^ *keep_stream (both|out|err) "\$work/(t02\.out|t07\.out|t16\.out|t17\.out|t18-\$t18n\.err)" ' "$here/no-docker.test.sh") +t43_tees=$(grep -vE '^[[:space:]]*#' "$here/no-docker.test.sh" | grep -cE '[|][[:space:]]*tee[[:space:]]') +[ "$t43_sites" -eq 5 ] && [ "$t43_tees" -eq 3 ] +result "T-43 the launches of T-02, T-07, T-16, T-17 and T-18 keep their streams through keep_stream; no other tee pipeline" $? \ + "($t43_sites sites, $t43_tees tee pipelines)" + +# --- T-44 L-15c's result lines (pure, both platforms): l15c_result reports +# each launch's own verdict: at all six moments a "yes" passes and a "no" +# fails with the status, each under the exact name of its moment, written +# out here. The form of 88b9085 to 5bcdd1b (the verdict tested, then a +# name holding a command substitution, then $?) reports that +# substitution's status instead and is caught. L-15c calls l15c_result +# once. Run in subshells: the suite's counts do not change. +t44_want=$( + for n in "0 ms: an allowed outcome" "20 ms: an allowed outcome" "50 ms: an allowed outcome" \ + "100 ms: an allowed outcome" "200 ms: an allowed outcome" \ + "300 ms: an allowed outcome; no profile of these launches is left"; do + printf 'PASS L-15c TERM at %s\nFAIL L-15c TERM at %s (rc 143)\n' "$n" "$n" + done +) +t44_old() { # MS OK RC: the earlier form + [ "$2" = yes ] + result "L-15c TERM at $1 ms: an allowed outcome$([ "$1" = 300 ] && echo '; no profile of these launches is left')" $? \ + "(rc $3)" +} +t44_all() { # FUNCTION: its lines for each moment, verdict yes then no + ( + for ms in 0 20 50 100 200 300; do + for ok in yes no; do "$1" "$ms" "$ok" 143; done + done + ) +} +t44_got=$(t44_all l15c_result) +t44_was=$(t44_all t44_old) +[ "$t44_got" = "$t44_want" ] && [ "$t44_was" != "$t44_want" ] && + [ "$(grep -c '^ l15c_result "\$ms" "\$l15c_ok" "\$r"$' "$here/no-docker.test.sh")" -eq 1 ] +result "T-44 L-15c reports each launch's own verdict: at all six moments a true one passes and a false one fails, under its exact name; the form whose name set the status is caught" $? \ + "($(printf '%s\n' "$t44_got" | grep -c '^PASS') of 12 passed, $(printf '%s\n' "$t44_was" | grep -c '^PASS') with the earlier form)" + +# --- T-45 L-07's caller (pure, both platforms): with a stub in place of +# no-docker.sh, l07_py gives it, in the pty case, a terminal on fd 1 that +# it can write (the master open and drained) and, in the socketpair case, +# a socket; in both no descriptor above 2, and the stub's status (125) +# comes back. The stub (a /bin/sh script that execs python3) lists its own +# descriptors, so a descriptor closed before or by its exec is not counted +# as one it had. The fixture of 88b9085 to 5bcdd1b, held here, is caught: +# its terminal was hung up, so the stub cannot write it. L-07 uses l07_py. +t45=$work/t45 +mkdir -p "$t45" || exit 2 +printf '%s\n' ' +import errno, os, stat, sys +fddir = "/proc/self/fd" if os.path.isdir("/proc/self/fd") else "/dev/fd" +fds = [] +for n in sorted(int(x) for x in os.listdir(fddir)): + try: + os.fstat(n) + except OSError as e: + if e.errno != errno.EBADF: + raise + continue + fds.append(str(n)) +kind = "tty" if os.isatty(1) else "socket" if stat.S_ISSOCK(os.fstat(1).st_mode) else "other" +try: + os.write(1, b"t45\n") + wrote = "ok" +except OSError as e: + wrote = errno.errorcode.get(e.errno, str(e.errno)) +sys.stderr.write("t45: fd 1 %s, write %s, fds %s\n" % (kind, wrote, " ".join(fds))) +sys.exit(125)' >"$t45/stub.py" && + printf '#!/bin/sh\nexec python3 -I "$0.py"\n' >"$t45/stub" && chmod 700 "$t45/stub" || exit 2 +t45_old=' +import os, pty, socket, sys +if sys.argv[1] == "socketpair": + a, b = socket.socketpair() + os.dup2(a.fileno(), 1, inheritable=True) +else: + m, s = pty.openpty() + os.dup2(s, 1, inheritable=True) +os.execv(sys.argv[2], [sys.argv[2], "/usr/bin/true"]) +' +t45_run() { # FIXTURE KIND NAME: " "; streams in NAME.out, NAME.err + local r + python3 -I -c "$1" "$2" "$t45/stub" "$t45/$3.out" 2>"$t45/$3.err" + r=$? + echo "$r $(grep '^t45: ' "$t45/$3.err")" +} +t45_p=$(t45_run "$l07_py" pty pty) +t45_s=$(t45_run "$l07_py" socketpair socketpair) +t45_o=$(t45_run "$t45_old" pty pty-earlier) +[ "$t45_p" = "125 t45: fd 1 tty, write ok, fds 0 1 2" ] && + case $t45_s in "125 t45: fd 1 socket, write "*", fds 0 1 2") true ;; *) false ;; esac && + [ "$t45_o" != "$t45_p" ] && [ "$(grep -c '^ python3 -I -c "\$l07_py" "\$kind" "\$nd" ' "$here/no-docker.test.sh")" -eq 1 ] +result "T-45 L-07's caller gives the launcher a live terminal or a socket on fd 1 and no descriptor above 2; the fixture whose terminal was hung up is caught" $? \ + "(pty: $t45_p; socketpair: $t45_s; earlier pty: $t45_o)" + +# --- T-46 L-18's count of its 64 KiB stderr line (pure, both platforms): +# l18_lines counts the complete lines that are exactly 65535 times "e" and +# a newline, byte for byte: one among other lines, and one as the last +# line, count 1; one of 65534 or 65536, or with one other byte, 0; two, 2. +# A file whose last line has no newline gives no count ("unterminated"), +# also when a complete line came before it; a missing file and a directory +# give "-". L-18 uses it. What this platform's grep makes of the earlier +# pattern '^e\{65535\}$' on the first fixture is printed as information +# (GNU grep refuses that bound, so on Linux the count was empty and L-18 +# could not pass). +t46=$work/t46 +mkdir -p "$t46/dir" || exit 2 +python3 -I -c ' +import sys +d, e = sys.argv[1], b"e" * 65535 +for name, data in (("one", b"x\n" + e + b"\ny\n"), ("end", b"x\n" + e + b"\n"), ("short", e[1:] + b"\n"), + ("long", e + b"e\n"), ("altered", e[:30000] + b"f" + e[30001:] + b"\n"), + ("two", e + b"\n" + e + b"\n"), ("last", b"x\n" + e), ("partial", e + b"\n" + e)): + with open(d + "/" + name, "wb") as f: + f.write(data) +' "$t46" || exit 2 +t46_got="" +for f in one end short long altered two last partial missing dir; do t46_got="$t46_got $f=$(l18_lines "$t46/$f")"; done +echo "INFO T-46 the earlier count on the first fixture here: $(grep -ac '^e\{65535\}$' "$t46/one" 2>&1 | head -n 1) ($(grep --version 2>&1 | head -n 1))" +[ "$t46_got" = " one=1 end=1 short=0 long=0 altered=0 two=2 last=unterminated partial=unterminated missing=- dir=-" ] && + [ "$(grep -c '^ l18_e=$(l18_lines "$work/l18.err")$' "$here/no-docker.test.sh")" -eq 1 ] +result "T-46 L-18's 64 KiB stderr line is counted byte for byte with no pattern bound: exact lines count, shorter, longer and altered ones do not" $? \ + "($t46_got)" + +# --- T-47 T-07's verdict (pure, both platforms): t07_verdict passes a +# launch that ended 0 and printed exactly "refused reached" between its +# workflow command lines; the same output with status 125, another output +# with status 0, or no output fails, and so does the right text from a +# filter that ended 2 (a grep shell function of a subshell of this case +# prints it and returns 2). T-07 takes the status from keep_stream (ks_rc) +# right after its launch and gives it to t07_verdict. The earlier form, +# which took grep's status for the launch's, passes the status-125 case +# and is caught. +t47=$work/t47 +mkdir -p "$t47" || exit 2 +printf '::stop-commands::0123\nrefused reached\n\n::0123::\n' >"$t47/right" && + printf '::stop-commands::0123\nreached reached\n\n::0123::\n' >"$t47/wrong" || exit 2 +t47_v() { if t07_verdict "$@" 2>/dev/null; then echo pass; else echo fail; fi; } +t47_old() { # STATUS FILE: the form of a10c6ea to 5bcdd1b + local r + t07=$(grep -v '^::' "$2") + r=$? + if [ "$r" -eq 0 ] && [ "$t07" = "refused reached" ]; then echo pass; else echo fail; fi +} +t47_got="$(t47_v 0 "$t47/right") $(t47_v 125 "$t47/right") $(t47_v 0 "$t47/wrong") $(t47_v 0 "$t47/none")" +t47_got="$t47_got $( + # shellcheck disable=SC2329 # called by t07_verdict, through t47_v + grep() { printf 'refused reached\n'; return 2; } + t47_v 0 "$t47/right" +)" +t47_was=$(t47_old 125 "$t47/right") +[ "$t47_got" = "pass fail fail fail fail" ] && [ "$t47_was" = pass ] && + [ "$(sed -n '/^t07_rc=\$ks_rc$/{x;p;q;};h' "$here/no-docker.test.sh")" = "' \"\$(cat \"\$work/t07.port\" 2>/dev/null || echo 0)\"" ] && + [ "$(grep -c '^t07_rc=\$ks_rc$' "$here/no-docker.test.sh")" -eq 1 ] && + [ "$(grep -c '^t07_verdict "\$t07_rc" "\$work/t07.out" && \[ -s "\$work/t07.port" \]$' "$here/no-docker.test.sh")" -eq 1 ] +result "T-47 T-07 needs its launch's own status 0 and exactly its output: that output with status 125, another output or none fail; the form that took grep's status is caught" $? \ + "($t47_got; earlier form with 125: $t47_was)" + +# --- T-38 the AppArmor lifecycle helper no-docker-aa.py as pure functions, on +# both platforms: fixtures only (every call that would open, list or write a +# file, read a link, run a program, open a pidfd or send a signal raises), +# its operations against a fake system (P-01 to P-11), the helper's +# functions called by what runs (W) and mutated copies of the helper +# rejected (M1 to M19). The numbers of assertions are fixed here. +python3 -I -B "$here/no-docker-aa-unit.py" "$root/scripts/ci/no-docker-aa.py" >"$work/T-38.out" 2>&1 +t38_rc=$? +t38=$(cat "$work/T-38.out") +printf '%s\n' "$t38" | grep -vE '^ok ' | sed 's/^/INFO /' +for want in P-01=8 P-02=7 P-03=13 P-04=7 P-05=10 P-06=15 P-07=6 P-08=4 P-09=6 P-10=4 P-11=6 W=9 M=19; do + tag=${want%=*} + n=$(printf '%s\n' "$t38" | grep -cE "^ok ${tag}[0-9]* ") + bad=$(printf '%s\n' "$t38" | grep -cE "^FAIL ${tag}[0-9]* ") + [ "$t38_rc" -eq 0 ] && [ "$n" -eq "${want#*=}" ] && [ "$bad" -eq 0 ] && + printf '%s\n' "$t38" | grep -qx 'SUMMARY 114 ok, 0 failed' + result "T-38 $tag: $n of ${want#*=} assertions pass" $? "(rc $t38_rc, $bad failed)" +done + +# --- T-19 CI order (static): no step that is not isolated follows an +# isolated one in a job; the required jobs exist; no if: conditions. +# On 38b5ebd: "Allocator unit tests" came before the real-Docker steps and +# the guard suites before ShellCheck and actionlint. +python3 -I "$here/ci-isolation-order.py" "$root" >"$work/T-19.out" 2>&1 +r=$? +t19=$(cat "$work/T-19.out") +printf '%s\n' "$t19" | grep -v '^ci-isolation-order: ok' | sed 's/^/INFO /' +[ "$r" -eq 0 ] +result "T-19 isolated steps are the last steps of their CI jobs; required jobs exist; no if: conditions" $? "(rc $r)" + +# --- T-19b the wrapper rule of that check, and T-19c the wrapper itself. +python3 -I -B "$here/ci-isolation-order-unit.py" "$here/ci-isolation-order.py" >"$work/T-19b.out" 2>&1 +r=$? +grep '^FAIL ' "$work/T-19b.out" | sed 's/^/INFO /' +[ "$r" -eq 0 ] +result "T-19b only the exact wrapper around an already isolated command counts as isolated" $? "(rc $r)" +python3 -I -B "$here/fdnorm-unit.py" "$root/scripts/ci/fdnorm.py" >"$work/T-19c.out" 2>&1 +r=$? +grep '^FAIL ' "$work/T-19c.out" | sed 's/^/INFO /' +[ "$r" -eq 0 ] +result "T-19c fdnorm.py leaves only fds 0 to 2 or refuses, and runs the command in its own process" $? "(rc $r)" + +# --- T-03/T-04 the harness against the mutant +check_safe() { # name + # Refused (125, the mutant never ran) or run isolated (97, every case ran + # and none reached an engine); either way the canary engine got nothing. + local cases answered + cases=$(count_in ' rc=' "$LOG") + answered=$(count_in 'canary-engine' "$LOG") + [ "$REQ" -eq 0 ] && [ "$answered" -eq 0 ] + result "$1: no request reached an engine" $? "(canary requests $REQ, answered cases $answered, rc $RC)" + [ "$STARTED" -eq 1 ] + result "$1: the harness was started" $? + if [ "$os" = Darwin ] && [ "$t0304" = bare ]; then + # The harness runs on the host and runs the mutant in its own sandbox. + [ "$RC" -eq 97 ] && [ "$cases" -ge 20 ] + result "$1: the mutant ran to its end inside the harness's isolation" $? "(rc $RC, $cases cases)" + else + # The canary is a reachable engine on Linux: the harness must refuse. + [ "$RC" -eq 125 ] && [ "$cases" -eq 0 ] && + grep -q '^no-docker: refused: inside no-docker.sh, but the isolation check fails' "$OUT" + result "$1: the harness refused before the mutant ran (an engine is reachable)" $? "(rc $RC, $cases cases)" + fi + [ "$(count_in '^real-socket-c[a-z]* rc=0 ' "$LOG")" -eq 0 ] + result "$1: the real Docker socket paths were not reached" $? + [ ! -e "$home_marker" ] + result "$1: nothing written under \$HOME" $? + rm -f "$home_marker" +} +run_case t03 "$harness" devenv "$t0304" +shown=("t03 $LOG $OUT") +check_safe "T-03 mutant as DEVENV" +run_case t04 "$harness" inplace "$t0304" +shown+=("t04 $LOG $OUT") +check_safe "T-04 mutant edited into bin/devenv" + +# --- T-05 control: the same check catches a harness without containment +run_case t05 "$ro/unit-uncontained.sh" inplace guard +shown+=("t05 $LOG $OUT") +answered=$(count_in 'canary-engine' "$LOG") +[ "$STARTED" -eq 1 ] && [ "$REQ" -gt 0 ] && [ "$answered" -gt 0 ] && [ "$RC" -eq 97 ] +result "T-05 control: without the containment the mutant's commands reach the canary engine" $? "(canary requests $REQ, answered cases $answered, rc $RC)" + +# --- L-16 the cost of a launch, measured last of all the cases that launch +# (after T-03 to T-05), so the count of launches is complete: 10 launches +# of true; c, the mean of their timing notices; the guard job projected as +# 184 s (its work besides launches, measured) + N * c, N = the 4 launches +# of the guard suites + this suite's. Above 540 s fails, compared exactly +# (l16_project, T-41; c and the projection are printed rounded up); above +# 360 s (the 6-minute target) is a warning only. +if [ "$os" = Linux ]; then + : >"$work/l16.out" + : >"$work/l16.err" + l16_bad=0 + for _ in 1 2 3 4 5 6 7 8 9 10; do + ndl /usr/bin/true >>"$work/l16.out" 2>>"$work/l16.err" || l16_bad=$((l16_bad + 1)) + done + l16_t=$(sed -n 's/^::notice title=no-docker::launch \([0-9]*\.[0-9]*\)s: .*/\1/p' "$work/l16.err") + l16_n=$(printf '%s\n' "$l16_t" | grep -c .) + [ "$l16_bad" -eq 0 ] && [ "$l16_n" -eq 10 ] + result "L-16 10 launches ran and timed themselves" $? "($l16_bad failed, $l16_n timings)" + l16_launches=$(($(wc -c <"$work/launch-count") + 4)) + # shellcheck disable=SC2086 # one word per timing + l16=$(l16_project "$l16_launches" $l16_t) + read -r l16_c l16_proj l16_verdict <<<"$l16" + echo "INFO L-16 c = ${l16_c:-?} s per launch, N = $l16_launches launches, projected guard job ${l16_proj:-?} s" + [ "$l16_verdict" = ok ] + result "L-16 the projected guard job is at most 540 s" $? "(${l16_proj:-?} s rounded up, $l16_verdict)" + case $l16_proj in + "" | *[!0-9]*) ;; + *) [ "$l16_proj" -le 360 ] || echo "::warning title=no-docker::the projected guard job ($l16_proj s) is above the 6-minute target" ;; + esac + [ "$(listed flowcrm-nd-)" = "$nd_listed_before" ] + result "L-16 no profile is left" $? +else + echo "SKIP L-16 (Linux CI only)" +fi + +echo +for s in "${shown[@]}"; do + read -r f log out <<<"$s" + printf 'INFO %s cases:\n' "$f" + ! regular "$log" || sed 's/^/INFO /' <"$log" 2>/dev/null + printf 'INFO %s harness output (last lines):\n' "$f" + tail -n 8 "$out" 2>/dev/null | sed 's/^/INFO /' +done +# The L-tests print a fixed number of result lines: one lost or added is a +# failure of the suite, not a count to adjust. +if [ "$os" = Linux ] && [ "$lcount" -ne 97 ]; then + echo "HARNESS ERROR: $lcount L-test result lines, not 97" + fail=$((fail + 1)) +fi +echo +echo "TOTAL $((pass + fail)) passed=$pass failed=$fail" +[ "$fail" -eq 0 ] && [ "$pass" -gt 0 ] diff --git a/scripts/ci/tests/no-docker/apparmor-negative.profile b/scripts/ci/tests/no-docker/apparmor-negative.profile new file mode 100644 index 0000000..983efb5 --- /dev/null +++ b/scripts/ci/tests/no-docker/apparmor-negative.profile @@ -0,0 +1,13 @@ +abi , +profile @NAME@ flags=(enforce,attach_disconnected) { + file, + unix, + network, + signal, + ptrace (read, readby, trace, tracedby) peer=@NAME@, + deny capability, + deny mount, + deny umount, + deny pivot_root, + deny network vsock, +} diff --git a/scripts/ci/tests/no-docker/canary.py b/scripts/ci/tests/no-docker/canary.py new file mode 100644 index 0000000..c20d378 --- /dev/null +++ b/scripts/ci/tests/no-docker/canary.py @@ -0,0 +1,34 @@ +import json, os, socketserver, sys +path, log = sys.argv[1], sys.argv[2] +class H(socketserver.StreamRequestHandler): + def handle(self): + while True: + line = self.rfile.readline() + if not line: + return + headers = {} + while True: + h = self.rfile.readline() + if h in (b"\r\n", b"\n", b""): + break + k, _, v = h.decode("latin-1").partition(":") + headers[k.strip().lower()] = v.strip() + if headers.get("user-agent") != "no-docker-probe": + with open(log, "a") as f: + f.write(line.decode("latin-1").strip() + " ua=" + headers.get("user-agent", "-") + "\n") + target = line.split()[1].decode("latin-1") if len(line.split()) > 1 else "/" + if target.endswith("/_ping"): + body = b"OK" + elif target.endswith("/version"): + body = json.dumps({"Version": "canary-engine", "ApiVersion": "1.45", "MinAPIVersion": "1.24", + "Os": "linux", "Arch": "amd64"}).encode() + else: + body = json.dumps({"message": "canary-engine"}).encode() + self.wfile.write(b"HTTP/1.1 200 OK\r\nApi-Version: 1.45\r\nOstype: linux\r\n" + b"Content-Type: application/json\r\nContent-Length: %d\r\n\r\n" % len(body)) + if not line.startswith(b"HEAD"): + self.wfile.write(body) + self.wfile.flush() +class S(socketserver.ThreadingMixIn, socketserver.UnixStreamServer): + daemon_threads = True +S(path, H).serve_forever() diff --git a/scripts/ci/tests/no-docker/drive.sh b/scripts/ci/tests/no-docker/drive.sh new file mode 100644 index 0000000..54ea1c2 --- /dev/null +++ b/scripts/ci/tests/no-docker/drive.sh @@ -0,0 +1,16 @@ +canary_py=$1 data=$2 canary=$3 +shift 3 +mkdir -p "$data" || exit 2 +: >"$data/canary.log" || exit 2 +rm -f "$canary" +python3 -I "$canary_py" "$canary" "$data/canary.log" & +cpid=$! +for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do [ -S "$canary" ] && break; sleep 0.2; done +[ -S "$canary" ] || { echo "HARNESS ERROR: the canary engine did not start"; kill "$cpid"; exit 2; } +: >"$data/drive-started" +DOCKER_HOST=unix://$canary "$@" +rc=$? +sleep 0.5 # let the canary finish writing +kill "$cpid" 2>/dev/null +wait "$cpid" 2>/dev/null +exit "$rc" diff --git a/scripts/ci/tests/no-docker/payload b/scripts/ci/tests/no-docker/payload new file mode 100644 index 0000000..0fcfeb3 --- /dev/null +++ b/scripts/ci/tests/no-docker/payload @@ -0,0 +1,37 @@ +qa_log=@LOG@ qa_canary=@CANARY@ qa_cfg=@CFG@ qa_marker=@MARKER@ +qa() { # case command... + qa_n=$1 + shift + qa_out=$("$@" 2>&1) + printf '%s rc=%s out=%s\n' "$qa_n" "$?" "$(printf '%s' "$qa_out" | tr '\n' ' ' | cut -c1-200)" >>"$qa_log" +} +qa_fmt='{{.Server.Version}}' +qa inherited-env docker version --format "$qa_fmt" +qa path-lookup docker -H "unix://$qa_canary" version --format "$qa_fmt" +for qa_p in /usr/local/bin/docker /usr/bin/docker /opt/homebrew/bin/docker /Applications/Docker.app/Contents/Resources/bin/docker; do + [ ! -x "$qa_p" ] || qa "absolute-path:$qa_p" "$qa_p" -H "unix://$qa_canary" version --format "$qa_fmt" +done +qa builtin-command command docker -H "unix://$qa_canary" version --format "$qa_fmt" +qa env-wrapper env docker -H "unix://$qa_canary" version --format "$qa_fmt" +qa env-docker-host env DOCKER_HOST="unix://$qa_canary" docker version --format "$qa_fmt" +qa context env DOCKER_CONFIG="$qa_cfg" docker --context canary version --format "$qa_fmt" +qa nested-bash bash -c 'docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" +qa nested-sh sh -c 'docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" +qa_x=$(docker -H "unix://$qa_canary" version --format "$qa_fmt" 2>&1) +printf 'command-substitution rc=%s out=%s\n' "$?" "$(printf '%s' "$qa_x" | tr '\n' ' ')" >>"$qa_log" +qa_x=`docker -H "unix://$qa_canary" version --format "$qa_fmt" 2>&1` +printf 'backticks rc=%s out=%s\n' "$?" "$(printf '%s' "$qa_x" | tr '\n' ' ')" >>"$qa_log" +qa_dk=docker +qa variable "$qa_dk" -H "unix://$qa_canary" version --format "$qa_fmt" +qa quoted "docker" -H "unix://$qa_canary" version --format "$qa_fmt" +qa exec-subshell sh -c 'exec docker -H "unix://$1" version --format "$2"' _ "$qa_canary" "$qa_fmt" +qa xargs sh -c 'echo version | xargs docker -H "unix://$1"' _ "$qa_canary" +(docker -H "unix://$qa_canary" version --format "$qa_fmt" >"$qa_log.bg" 2>&1; echo "background rc=$? out=$(tr '\n' ' ' <"$qa_log.bg")" >>"$qa_log") & +wait +qa sudo sudo -n docker -H "unix://$qa_canary" version --format "$qa_fmt" +qa curl-unix-socket curl -sS --max-time 5 --unix-socket "$qa_canary" http://docker/version +qa python-raw-socket python3 -c 'import socket,sys; s=socket.socket(socket.AF_UNIX); s.connect(sys.argv[1]); s.sendall(b"GET /version HTTP/1.0\r\n\r\n"); print(s.recv(4096).decode())' "$qa_canary" +qa real-socket-cli docker -H unix:///var/run/docker.sock version --format "$qa_fmt" +qa real-socket-curl curl -sS --max-time 5 --unix-socket /var/run/docker.sock http://docker/_ping +qa home-write touch "$qa_marker" +exit 97 diff --git a/tests/devenv/integration.sh b/tests/devenv/integration.sh index f67aec8..556e9ab 100755 --- a/tests/devenv/integration.sh +++ b/tests/devenv/integration.sh @@ -8,7 +8,10 @@ # # Usage: tests/devenv/integration.sh # DEVENV_TEST_PORT_BASE first port of the 20-port test range (default 8080) -# DEVENV bin/devenv copy to test (default: this checkout's) +# DEVENV accepted only if identical to this checkout's +# bin/devenv: this suite uses the real Docker daemon, +# so a modified or mutated copy belongs in +# tests/devenv/unit.sh, which runs without one # DEVENV_TEST_SCENARIOS space-separated subset, e.g. "pa16 pa21" (default: # all); the summary names the scenarios that ran # Needs: docker with Compose v2, python3 and curl (test tools only). @@ -23,6 +26,10 @@ set -u here=$(cd -P "$(dirname "$0")" && pwd -P) repo=$(cd -P "$here/../.." && pwd -P) +if [ -n "${DEVENV-}" ] && ! cmp -s "$DEVENV" "$repo/bin/devenv"; then + echo "REFUSED: DEVENV=$DEVENV differs from $repo/bin/devenv; this suite uses the real Docker daemon, run tests/devenv/unit.sh for a modified copy" >&2 + exit 2 +fi BASE=${DEVENV_TEST_PORT_BASE:-8080} LO=$BASE HI=$((BASE + 19)) @@ -97,7 +104,7 @@ new_checkout() { # name -> sets CO CO=$work/$1 mkdir -p "$CO/bin" cp "$repo/Makefile" "$repo/compose.fixture.yaml" "$CO/" - cp "${DEVENV:-$repo/bin/devenv}" "$CO/bin/devenv" + cp "$repo/bin/devenv" "$CO/bin/devenv" printf '%s\n' "$CO" >>"$work/checkouts" } diff --git a/tests/devenv/unit.sh b/tests/devenv/unit.sh index f786978..c6e4ec9 100755 --- a/tests/devenv/unit.sh +++ b/tests/devenv/unit.sh @@ -19,6 +19,11 @@ set -u here=$(cd -P "$(dirname "$0")" && pwd -P) +# The suite sources and runs bin/devenv, or a modified or mutated copy of +# it, so it runs only where no container engine can be reached. +# NO_DOCKER_WORK= is the isolation's only writable directory (--work). +"$here/../../scripts/ci/no-docker.sh" --check >/dev/null 2>&1 || + exec "$here/../../scripts/ci/no-docker.sh" ${NO_DOCKER_WORK:+--work "$NO_DOCKER_WORK"} "$BASH" "$here/unit.sh" "$@" # DEVENV= runs the suite against another copy (mutation checks). devenv_script=${DEVENV:-"$here/../../bin/devenv"} # shellcheck source=../../bin/devenv @@ -109,6 +114,10 @@ expect_eq "U-03 all 20 range ports considered" 20 "$#" RUNTIME_DIR=$work/rt STATE_FILE=$RUNTIME_DIR/dev.env LOCK_DIR=$RUNTIME_DIR/lock +# Every runtime path, so that nothing is written into the checkout's .runtime. +SECRETS_DIR=$RUNTIME_DIR/secrets +GATE_DIR=$RUNTIME_DIR/gate +GATE_FILE=$GATE_DIR/security-gate mkdir -p "$RUNTIME_DIR" # Fixtures are written to $work/st first: piping into state_case would run it # in a subshell and lose its pass/fail counts. @@ -271,30 +280,23 @@ result "U-09 no volume removal or global prune in bin/devenv" $r # bin/devenv, then runs MODE: "hold" waits for GO itself; the other modes # run a script from $work through run_op (recorded, own process group). # await.sh READY GO: create READY, then wait (at most 30 s) for GO. -cat >"$work/await.sh" <<'AWAIT' -: >"$1" +printf '%s\n' ': >"$1" i=0 -while [ ! -e "$2" ] && [ "$i" -lt 300 ]; do sleep 0.1; i=$((i + 1)); done -AWAIT +while [ ! -e "$2" ] && [ "$i" -lt 300 ]; do sleep 0.1; i=$((i + 1)); done' >"$work/await.sh" # tree.sh READY GO GCPID: like a Docker CLI with its Compose plugin, start a # descendant that waits for GO, then wait for it. -cat >"$work/tree.sh" <<'TREE' -sh "${0%/*}/await.sh" "$1.gc" "$2" & +printf '%s\n' 'sh "${0%/*}/await.sh" "$1.gc" "$2" & echo $! >"$3" i=0 while [ ! -e "$1.gc" ] && [ "$i" -lt 100 ]; do sleep 0.1; i=$((i + 1)); done : >"$1" -wait -TREE +wait' >"$work/tree.sh" # termchild.sh READY MARK: record a received SIGTERM in MARK, then exit. -cat >"$work/termchild.sh" <<'TERMCHILD' -trap ': >"$2"; exit 0' TERM +printf '%s\n' 'trap '\'': >"$2"; exit 0'\'' TERM : >"$1" i=0 -while [ "$i" -lt 300 ]; do sleep 0.1; i=$((i + 1)); done -TERMCHILD -cat >"$work/lockctl.sh" <<'LOCKCTL' -# lockctl.sh DEVENV RUNTIME_DIR LOCK_WAIT [hold|child|tree|term ARGS...] +while [ "$i" -lt 300 ]; do sleep 0.1; i=$((i + 1)); done' >"$work/termchild.sh" +printf '%s\n' '# lockctl.sh DEVENV RUNTIME_DIR LOCK_WAIT [hold|child|tree|term ARGS...] . "$1" SELF=$1 RUNTIME_DIR=$2 @@ -302,15 +304,14 @@ LOCK_DIR=$2/lock LOCK_WAIT=$3 work=${0%/*} trap on_exit EXIT -trap 'on_signal 143 TERM' TERM +trap '\''on_signal 143 TERM'\'' TERM acquire_lock test case ${4-} in hold) sh "$work/await.sh" "$5" "$6" ;; child) run_op sh "$work/await.sh" "$5" "$6" ;; tree) run_op sh "$work/tree.sh" "$5" "$6" "$7" ;; term) run_op sh "$work/termchild.sh" "$5" "$6" ;; -esac -LOCKCTL +esac' >"$work/lockctl.sh" # An array, not a function: env and & must start the lock owner itself. lockctl=("$BASH" "$work/lockctl.sh" "$devenv_script" "$RUNTIME_DIR") wait_file() { # file: bounded wait (10 s) for a readiness signal @@ -480,16 +481,14 @@ result "U-11 compose_op() refuses to run without the lock" $? "(rc $r: $out)" # try_edge() with a fake docker whose `compose up` prints a bind conflict and # then exits 1 or is killed by SIGKILL; `compose ps` reports no edge. mkdir -p "$work/fakeedge" -cat >"$work/fakeedge/docker" <<'FAKE' -#!/bin/sh +printf '%s\n' '#!/bin/sh case " $* " in *" up "*) - echo 'Error response from daemon: Bind for 127.0.0.1:8080 failed: port is already allocated' >&2 + echo '\''Error response from daemon: Bind for 127.0.0.1:8080 failed: port is already allocated'\'' >&2 [ "$FAKE_MODE" = killed ] && kill -KILL $$ exit 1 ;; esac -exit 0 -FAKE +exit 0' >"$work/fakeedge/docker" chmod +x "$work/fakeedge/docker" # shellcheck disable=SC2030,SC2031 # PATH and globals change only inside the subshell, on purpose edge_rc() { # mode: try_edge result with the fake docker @@ -691,15 +690,14 @@ result "U-15 the URL label follows the stack" $? # http_ok requests the stack's readiness path and requires the stack's own # body: one local server answers like the application (/health/ready) and # the fixture (/health); another answers every path with an SPA page. -cat >"$work/ready_server.py" <<'PYSERVER' -import http.server, sys +printf '%s\n' 'import http.server, sys mode = sys.argv[2] class H(http.server.BaseHTTPRequestHandler): def do_GET(self): if mode == "spa": status, body = 200, b"
" elif self.path == "/health/ready": - status, body = 200, b'{"status":"ready"}' + status, body = 200, b'\''{"status":"ready"}'\'' elif self.path == "/health": status, body = 200, b"ok\n" else: @@ -712,8 +710,7 @@ class H(http.server.BaseHTTPRequestHandler): pass server = http.server.HTTPServer(("127.0.0.1", 0), H) open(sys.argv[1], "w").write(str(server.server_address[1])) -server.serve_forever() -PYSERVER +server.serve_forever()' >"$work/ready_server.py" python3 "$work/ready_server.py" "$work/ready.port" stacks & bg_pids="$bg_pids $!" python3 "$work/ready_server.py" "$work/spa.port" spa & @@ -1092,8 +1089,7 @@ RUNTIME_DIR=$saved_runtime GATE_FILE=$saved_gate GATE_DIR=$saved_gate_dir # --- U-21 gate_blocks reads the edge's real answer (FND-009E) # Against one-shot local listeners: only the gate's own 503 counts as # blocked, not any other 503 and not a 200. -cat >"$work/u21-serve.py" <<'PY' -import socket, sys +printf '%s\n' 'import socket, sys status, body = sys.argv[2], sys.argv[3] s = socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(("127.0.0.1", 0)); s.listen(1) @@ -1115,8 +1111,7 @@ try: pass except OSError: pass -c.close(); s.close() -PY +c.close(); s.close()' >"$work/u21-serve.py" u21() { # name status body expected(0 blocked|1 not) local pf=$work/u21.port i=0 r rm -f "$pf" @@ -1405,22 +1400,21 @@ u25 "a missing password file is not a login" row "$work/u25.missing" no-file/fal # stack is refused. remove_untagged_images may remove only the untagged # images labelled with the run's project, by ID and never forced. mkdir -p "$work/fakeimg" -cat >"$work/fakeimg/docker" <<'FAKE' -#!/bin/sh +printf '%s\n' '#!/bin/sh d=$FAKE_IMG echo "$*" >>"$d/calls" -named() { awk -v i="$1" '$2 == i { f = 1 } END { exit !f }' "$d/names"; } -used() { awk -v i="$1" '$2 == i { f = 1 } END { exit !f }' "$d/containers"; } -drop_image() { awk -v i="$1" '$1 != i' "$d/images" >"$d/images.new" && mv "$d/images.new" "$d/images"; } +named() { awk -v i="$1" '\''$2 == i { f = 1 } END { exit !f }'\'' "$d/names"; } +used() { awk -v i="$1" '\''$2 == i { f = 1 } END { exit !f }'\'' "$d/containers"; } +drop_image() { awk -v i="$1" '\''$1 != i'\'' "$d/images" >"$d/images.new" && mv "$d/images.new" "$d/images"; } case "$1 $2" in - "ps -aq"*) cut -d' ' -f1 "$d/containers" ;; + "ps -aq"*) cut -d'\'' '\'' -f1 "$d/containers" ;; "inspect -f") shift 3 - for c in "$@"; do awk -v c="$c" '$1 == c { print $2 }' "$d/containers"; done ;; + for c in "$@"; do awk -v c="$c" '\''$1 == c { print $2 }'\'' "$d/containers"; done ;; "image ls") shift 2 case $1 in - --format) cut -d' ' -f1 "$d/names"; exit 0 ;; + --format) cut -d'\'' '\'' -f1 "$d/names"; exit 0 ;; esac # image IDs, filtered like Docker: dangling=true (no name) and # label=com.docker.compose.project=P. IDs in "stale" are listed as @@ -1441,36 +1435,36 @@ case "$1 $2" in done <"$d/images" ;; "image inspect") ref=$5 - id=$(awk -v r="$ref" '$1 == r { print $2 }' "$d/names") - [ -n "$id" ] || id=$(awk -v r="$ref" '$1 == r { print $1 }' "$d/images") + id=$(awk -v r="$ref" '\''$1 == r { print $2 }'\'' "$d/names") + [ -n "$id" ] || id=$(awk -v r="$ref" '\''$1 == r { print $1 }'\'' "$d/images") [ -n "$id" ] || exit 1 case $4 in - *RepoTags*) awk -v i="$id" '$2 == i { print $1 }' "$d/names" ;; + *RepoTags*) awk -v i="$id" '\''$2 == i { print $1 }'\'' "$d/names" ;; *) echo "$id" ;; esac ;; "image rm") shift 2 force=0 - case " $* " in *' -f '* | *' --force '*) force=1 ;; esac + case " $* " in *'\'' -f '\''* | *'\'' --force '\''*) force=1 ;; esac for a in "$@"; do case $a in -f | --force) continue ;; esac - id=$(awk -v r="$a" '$1 == r { print $2 }' "$d/names") + id=$(awk -v r="$a" '\''$1 == r { print $2 }'\'' "$d/names") if [ -n "$id" ] && [ "$force" = 0 ]; then # By name: only this name goes, and the image with its last name; # refused for the last name of an image a container uses. - if [ "$(awk -v i="$id" '$2 == i' "$d/names" | wc -l)" -eq 1 ] && used "$id"; then + if [ "$(awk -v i="$id" '\''$2 == i'\'' "$d/names" | wc -l)" -eq 1 ] && used "$id"; then echo "conflict: image is being used by a container" >&2 exit 1 fi - awk -v r="$a" '$1 != r' "$d/names" >"$d/names.new" && mv "$d/names.new" "$d/names" + awk -v r="$a" '\''$1 != r'\'' "$d/names" >"$d/names.new" && mv "$d/names.new" "$d/names" named "$id" || drop_image "$id" continue fi # By ID or forced: every name of the image goes, and the image. [ -n "$id" ] || id=$a - if ! named "$id" && ! awk -v i="$id" '$1 == i { f = 1 } END { exit !f }' "$d/images"; then + if ! named "$id" && ! awk -v i="$id" '\''$1 == i { f = 1 } END { exit !f }'\'' "$d/images"; then echo "No such image: $a" >&2 exit 1 fi @@ -1478,12 +1472,11 @@ case "$1 $2" in echo "conflict: unable to delete $id (must be forced)" >&2 exit 1 fi - awk -v i="$id" '$2 != i' "$d/names" >"$d/names.new" && mv "$d/names.new" "$d/names" + awk -v i="$id" '\''$2 != i'\'' "$d/names" >"$d/names.new" && mv "$d/names.new" "$d/names" drop_image "$id" done ;; *) echo "unexpected: docker $*" >&2; exit 99 ;; -esac -FAKE +esac' >"$work/fakeimg/docker" chmod +x "$work/fakeimg/docker" u26_dev=$( . "$here/../app/lib.sh" && project_of "$APP_REPO") u26_run=flowcrm-aaaaaa-test-bbbbbb @@ -1655,8 +1648,7 @@ u26_guard "no prune command of any kind" "$u26_pat_prune" # removal, one after && does not. (The command name is in a variable so that # this file has no match.) u26_dk=docker -cat >"$work/u26-fixture.sh" <&1 -f $u26_dk image rm y &>/dev/null --force -$u26_dk image rm y && rm -f z -EOF +$u26_dk image rm y && rm -f z" >"$work/u26-fixture.sh" u26_hits() { # pattern: first line numbers of the fixture's matching lines u26_lines "$work/u26-fixture.sh" | grep -E -- "$1" | cut -d: -f2 | paste -sd ' ' - }