From dae513e6a4840ab3f4c8cdc724146d169f0ac4d2 Mon Sep 17 00:00:00 2001 From: yuanhe Date: Sat, 3 Oct 2026 19:26:15 +0800 Subject: [PATCH 1/2] Bundle Compose init metadata --- deploy/README.md | 2 +- deploy/compose/compose.yaml | 14 +-- deploy/compose/test_compose.py | 5 +- deploy/distribution/Ingress.Dockerfile | 10 +- deploy/install.dev.sh | 13 ++- docs/getting-started/install-options.md | 4 +- docs/maintainers.md | 6 +- docs/zh/getting-started/install-options.md | 6 +- docs/zh/maintainers.md | 8 +- scripts/build-core-distribution.sh | 19 ++-- scripts/compose-smoke.py | 41 ++++++-- scripts/core-distribution-manifest.py | 24 ++++- scripts/core-distribution-manifest.test.py | 18 +++- scripts/publish-core-release.py | 3 +- scripts/render-compose.py | 12 +-- services/core/cmd/oac/init.go | 105 +++++++-------------- services/core/cmd/oac/init_test.go | 85 +++++++++-------- services/core/cmd/oac/main.go | 2 +- 18 files changed, 208 insertions(+), 169 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 7d254a21..a69bfa3f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -11,7 +11,7 @@ `install.sh` downloads its release's `compose.yaml` and port files, checks them against `compose-sha256sums.txt`, writes `.env`, and starts Compose. Core applies database migrations when it starts. The host needs Linux amd64 and Docker Compose 2.26 or newer. [Configuration](../docs/configuration.md) owns the installation layout and settings. -`oac` is a Go command (`services/core/cmd/oac`) in the Core image and the ingress image. The host copy implements `apply`, `core-key` and `rotate-core-key`; `core-key --show` runs `oac-web core-key` in the Web container. Start, stop, logs and removal are `docker compose`. `apply` runs `oac-core check-config` before recreating services. The ingress image runs data initialization as `oac init` and contains no Python. No service receives a Docker socket. +`oac` is a Go command (`services/core/cmd/oac`) in the Core image and the ingress image. The host copy implements `apply`, `core-key` and `rotate-core-key`; `core-key --show` runs `oac-web core-key` in the Web container. Start, stop, logs and removal are `docker compose`. `apply` runs `oac-core check-config` before recreating services. The ingress image runs data initialization as `oac init`, verifies and copies its bundled node metadata without network access, and contains no Python. No service receives a Docker socket. Web serves the console and forwards `/v1` and `/api/v1` to Core, so it is the only published service. HTTPS is terminated by the operator's reverse proxy or hosting platform, which routes to `web:8080`; `OAC_PUBLIC_URL` records that origin. diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml index dbe5cdd0..85240dec 100644 --- a/deploy/compose/compose.yaml +++ b/deploy/compose/compose.yaml @@ -1,11 +1,7 @@ -# Release template. scripts/render-compose.py fills the __OAC_*__ tokens with this -# release's source revision and node-metadata checksum. Images default to the -# floating latest tags; set OAC_IMAGE_CORE, OAC_IMAGE_WEB or OAC_IMAGE_INGRESS -# to select another reference. Do not run this file until it has been rendered. -# Data is bind-mounted from ${OAC_DATA_DIR:-./data}. Set OAC_PUBLIC_URL when the -# platform domain is ready; startup defaults to localhost. Other process -# settings pass through unchanged; Core owns their defaults. -x-ingress-image: &ingress-image ${OAC_IMAGE_INGRESS:-ghcr.io/minimax-ai/openagentcore/ingress:latest} +# Release template. The publisher pins the initialization image to this release. +# Core and Web default to latest; OAC_IMAGE_* selects another reference. +# Data is bind-mounted from ${OAC_DATA_DIR:-./data}. +x-ingress-image: &ingress-image ${OAC_IMAGE_INGRESS:-__OAC_INIT_IMAGE__} services: init: image: *ingress-image @@ -15,8 +11,6 @@ services: command: [/usr/local/bin/oac, init] environment: OAC_REVISION: __OAC_REVISION__ - OAC_RELEASE_BASE: __OAC_RELEASE_BASE__ - OAC_ARCHIVE_CHECKSUM: __OAC_ARCHIVE_CHECKSUM__ volumes: - type: bind source: ${OAC_DATA_DIR:-./data} diff --git a/deploy/compose/test_compose.py b/deploy/compose/test_compose.py index 29bb780d..f838f0ba 100644 --- a/deploy/compose/test_compose.py +++ b/deploy/compose/test_compose.py @@ -18,8 +18,7 @@ def rendered_compose(directory): text = render_compose.render({ 'REVISION': 'd' * 40, - 'RELEASE_BASE': 'https://example.com/releases/v1/', - 'ARCHIVE_CHECKSUM': 'e' * 64, + 'INIT_IMAGE': 'ghcr.io/minimax-ai/openagentcore/ingress@sha256:' + 'e' * 64, }) path = Path(directory) / 'compose.yaml' path.write_text(text) @@ -56,7 +55,7 @@ def test_compose_uses_private_services_and_ordered_initialization(self): for service in services.values(): self.assertNotIn('build', service) self.assertNotIn('ports', service) - self.assertTrue(service['image'].endswith(':latest') or service['image'] == 'postgres:16-alpine') + self.assertTrue(service['image'].endswith(':latest') or service['image'] == 'postgres:16-alpine' or service['image'].endswith('@sha256:' + 'e' * 64)) for volume in service.get('volumes', []): self.assertNotIn('docker.sock', json.dumps(volume)) self.assertEqual(volume['type'], 'bind') diff --git a/deploy/distribution/Ingress.Dockerfile b/deploy/distribution/Ingress.Dockerfile index bedebf58..0f2ef87a 100644 --- a/deploy/distribution/Ingress.Dockerfile +++ b/deploy/distribution/Ingress.Dockerfile @@ -1,8 +1,6 @@ -# One-time data initialization. oac init runs as root so it can chown data -# directories, then exits. It downloads the node payload over HTTPS, so the -# image carries CA certificates. scripts/build-core-distribution.sh builds this -# from a context that also contains the oac binary. -FROM alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8 -RUN apk add --no-cache ca-certificates +# One-time data initialization runs as root to prepare data ownership. +# Only the node installation metadata accompanies the oac binary. +FROM scratch COPY --chmod=0555 oac /usr/local/bin/oac +COPY --chmod=0444 node-payload/ /opt/oac/node-payload/ ENTRYPOINT [] diff --git a/deploy/install.dev.sh b/deploy/install.dev.sh index e6355526..31005d27 100755 --- a/deploy/install.dev.sh +++ b/deploy/install.dev.sh @@ -69,7 +69,15 @@ go_build() { go_build services/core/cmd/environment-key "$build/core/bin/oac-core-environment-key" go_build services/core/cmd/oac "$build/core/bin/oac" go_build services/web "$build/web/oac-web" - go_build services/core/cmd/oac "$build/ingress/oac" + payload_revision="$(python3 - "$build/ingress/node-payload" <<'PYCODE' +import importlib.util, pathlib, sys +spec = importlib.util.spec_from_file_location("smoke", "scripts/compose-smoke.py") +smoke = importlib.util.module_from_spec(spec) +spec.loader.exec_module(smoke) +print(smoke.prepare_pinned_payload(pathlib.Path(sys.argv[1]))) +PYCODE +)" + go build -trimpath -ldflags "-X main.buildRevision=$payload_revision" -o "$build/ingress/oac" ./services/core/cmd/oac ) mkdir -p "$build/core/e2b" "$build/core/native-installers" python3 - "$build/core/native-installers/catalog.json" "$revision" "$protocol" <<'PY' @@ -100,8 +108,7 @@ spec.loader.exec_module(render) pins = json.loads((root / "deploy/compose/smoke-pins.json").read_text()) (dest / "compose.yaml").write_text(render.render({ "REVISION": pins["revision"], - "RELEASE_BASE": pins["release_base"], - "ARCHIVE_CHECKSUM": pins["archive_checksum"], + "INIT_IMAGE": "ghcr.io/minimax-ai/openagentcore/ingress@sha256:" + "0" * 64, })) PY cp "$repo_root/deploy/compose/ports.yaml" "$install_dir/ports.yaml" diff --git a/docs/getting-started/install-options.md b/docs/getting-started/install-options.md index a9cd81cd..757ed171 100644 --- a/docs/getting-started/install-options.md +++ b/docs/getting-started/install-options.md @@ -14,7 +14,7 @@ With the one-line command, append them after `bash -s --`. `--version TAG` selec ## Docker Compose and hosting platforms -Use the `compose.yaml` from a release with Docker Compose 2.26 or newer on Linux amd64. The release renders node metadata into the [Compose template](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml). Core and Web use the `latest` images, and PostgreSQL uses `postgres:16-alpine`. It starts PostgreSQL, Core and Web. Web forwards `/v1` and `/api/v1` to Core. Data is bind-mounted from a directory. The one-time initialization service generates random secrets there and prepares the node installer; Core applies database migrations when it starts. [Compose configuration](../configuration.md#compose-installations) owns the settings and the data directory. +Use the `compose.yaml` from a release with Docker Compose 2.26 or newer on Linux amd64. The release pins its initialization image and source revision in the [Compose template](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml). Core and Web use the `latest` images, and PostgreSQL uses `postgres:16-alpine`. It starts PostgreSQL, Core and Web. Web forwards `/v1` and `/api/v1` to Core. Data is bind-mounted from a directory. The one-time initialization service generates random secrets there and prepares the node installer; Core applies database migrations when it starts. [Compose configuration](../configuration.md#compose-installations) owns the settings and the data directory. For a local trial, download `compose.yaml` and `ports.yaml` from the same release into one directory, then run: @@ -25,7 +25,7 @@ docker compose -f compose.yaml exec web oac-web core-key `oac-web core-key` prints the generated Core key to your terminal without writing it to container logs. Open `http://localhost:8080` and use that key to sign in. All installation secrets are generated automatically; keep the same Compose project and its data directory when restarting. -The first initialization downloads and verifies the release's approximately 385 MB control archive, retaining only the small node installation metadata. Later starts verify the saved files without downloading again. Image downloads are additional. An interrupted first initialization can be rerun; an existing database with missing installation secrets is refused. +The initialization image contains only the small node installation metadata alongside the initialization command. First startup verifies and copies that metadata without downloading the control archive or requiring access to GitHub Releases. Later starts verify the saved files. Container images still need to be pulled. An interrupted first initialization can be rerun; an existing database with missing installation secrets is refused. You can deploy before choosing a domain: leave `OAC_PUBLIC_URL` unset or empty, then follow [Compose configuration](../configuration.md#compose-installations) to set it and redeploy once the platform's domain is ready. The initial localhost origin allows services to start; Web accepts the configured host only, so platform-domain access becomes available after that redeployment. diff --git a/docs/maintainers.md b/docs/maintainers.md index 0f6e82a2..600a85ba 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -35,6 +35,8 @@ make build-core-distribution The build reuses the Core, Web, Runtime, SDK and helper builders. The manifest records the commit and source tree, image config and OCI manifest digests, the Runtime OCI manifest digest, the microsandbox runtime and firmware hashes, and the size and SHA-256 of every Runtime and node artifact; native installers carry only their SHA-256 in the [catalog](#native-installers). Output is the control archive and its `.sha256`, the optional offline archive, and the versioned Runtime, node and native installer assets. Nothing is published. Rebuilding into a directory that already holds this commit's distribution is refused. +The ingress image carries only `oac` and four node metadata files: `manifest.json`, `SHA256SUMS`, `node-install.pyz` and `runtime/seccomp.json`. The build prepares their manifest from the same Runtime and artifact identities as the distribution, then verifies initialization with networking disabled. The node manifest omits control-plane image identities so it can be packaged before the ingress image itself exists. + The control archive carries no Runtime image or node execution artifacts; the offline archive carries them. The [download contract](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/node/README.md#download-contract) describes how nodes obtain them. A distribution carries the docs listed in `BUNDLED_DOCS` in `scripts/core-distribution-manifest.py`. Links between bundled docs stay relative; every other relative link is rewritten to the same file on GitHub at the bundle's commit. The build fails when a link or anchor does not resolve, and `make check-distribution` runs the same check on every tracked Markdown file outside `example/`. Update the list when you add or move a doc that the installer or its output refers to. @@ -130,7 +132,7 @@ Distribution and Runtime archives use `pigz` level 6 with at most four compressi ### Container registry -Version releases and manual `build-` drafts publish Linux amd64 images as `ghcr.io/minimax-ai/openagentcore/:`, where `` is `core`, `web`, `runtime` or `ingress`. For example, `ghcr.io/minimax-ai/openagentcore/core:v1.2.3`. A draft uses the tag `build-`. PostgreSQL uses its upstream image and is not republished. The registry images are loaded from the release archives without rebuilding. Existing version tags are reused only when their image config digest matches the release; a different image stops publication. A stable release also moves each component's `latest` tag to that image. Prereleases and drafts leave `latest` unchanged. SemVer build metadata uses `_` in place of `+` in container tags; version strings longer than 128 characters cannot be published to GHCR. After the images are verified, the publisher uploads `compose.yaml` and `ports.yaml`, with checksums, rendered for that release. A draft Release stays unpublished. +Version releases and manual `build-` drafts publish Linux amd64 images as `ghcr.io/minimax-ai/openagentcore/:`, where `` is `core`, `web`, `runtime` or `ingress`. For example, `ghcr.io/minimax-ai/openagentcore/core:v1.2.3`. A draft uses the tag `build-`. PostgreSQL uses its upstream image and is not republished. The registry images are loaded from the release archives without rebuilding. Existing version tags are reused only when their image config digest matches the release; a different image stops publication. A stable release also moves each component's `latest` tag to that image. Prereleases and drafts leave `latest` unchanged. SemVer build metadata uses `_` in place of `+` in container tags; version strings longer than 128 characters cannot be published to GHCR. After the images are verified, the publisher uploads `compose.yaml` and `ports.yaml`, with checksums, rendered for that release. Compose pins the ingress image by its registry digest; initialization rejects an image whose build revision differs from the Compose revision. A draft Release stays unpublished. The combined build/publication job uses `GITHUB_TOKEN` with `packages: write`. On the first publication, GitHub creates each container package as private: a package administrator must change all four packages to **Public** in their package settings before users can pull anonymously. See [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). Verify an unauthenticated pull after changing visibility. Repository visibility alone does not make a new container package public. @@ -177,7 +179,7 @@ The planner compares the PR event's tested merge commit with its verified first `.github/actionlint.yaml` selects hygiene and lint. Known workflow changes select their consumers: the CI review and actionlint workflows run hygiene and lint; native workflow changes add native checks; API acceptance workflow changes add API checks with container acceptance enabled; website workflow changes add website checks. The shared Node action selects every job that uses it plus lint. A new or unclassified workflow/action selects the full gate until its consumers are declared in the planner. Planner tests and CI measurement scripts run hygiene; changing the planner itself runs the full gate. -Compose template and Compose test changes select both `distribution` fixtures and the `compose` smoke job; Core, Web, shared Go packages and the image Dockerfiles also select the smoke job. Run `python3 scripts/compose-smoke.py` locally with Docker available to repeat it. The script uses a unique project, an automatically assigned loopback port and artifacts under `~/.oac/tests/`; it removes its containers and volumes on exit. CI also performs cleanup after a failed or interrupted smoke step. Diagnostics show container status without printing HTTP response bodies or sign-in keys. Core, Web and the ingress image are built from the checkout; Web serves a placeholder page instead of the console build. Node metadata comes from the release pinned in `deploy/compose/smoke-pins.json`. This checks generic Compose behavior; it does not run a Dokploy/Coolify instance or execute a model. +Compose template and Compose test changes select both `distribution` fixtures and the `compose` smoke job; Core, Web, shared Go packages and the image Dockerfiles also select the smoke job. Run `python3 scripts/compose-smoke.py` locally with Docker available to repeat it. The script uses a unique project, an automatically assigned loopback port and artifacts under `~/.oac/tests/`; it removes its containers and volumes on exit. CI also performs cleanup after a failed or interrupted smoke step. Diagnostics show container status without printing HTTP response bodies or sign-in keys. Core, Web and the ingress image are built from the checkout; Web serves a placeholder page instead of the console build. Build-time node metadata comes from the release pinned in `deploy/compose/smoke-pins.json`; the initialization container runs with networking disabled. This checks generic Compose behavior; it does not run a Dokploy/Coolify instance or execute a model. Go module and workspace inputs select backend, API (including the container), native and distribution checks. Each Node module owns its manifest and lockfile. Website dependencies select website checks; Web dependencies select Web and browser checks; example dependencies select example checks; shared TypeScript client dependencies select Web, browser and example checks; Claude adapter dependencies select Harness, native and distribution checks. Shared package-manager configuration selects all Node consumers. The root TypeScript configuration selects Web and example checks; the adapter TypeScript configuration selects Harness and native checks. Each selected set includes hygiene. Mixed changes accumulate their consumers, and every job reads the same plan instead of maintaining its own path list. For example, a notification-only PR skips database, browser and native jobs, while a notification plus Core change adds backend and API checks. diff --git a/docs/zh/getting-started/install-options.md b/docs/zh/getting-started/install-options.md index d578ae6f..52e0a141 100644 --- a/docs/zh/getting-started/install-options.md +++ b/docs/zh/getting-started/install-options.md @@ -1,7 +1,7 @@ --- title: "安装选项与高级部署" source: docs/getting-started/install-options.md -source_hash: 4b847976f3b3bc4fe4afe7b7946c5815df9fecf75dc39648621814e793288f9e +source_hash: 93e842b2818adb1ca56bdca911fd0c828868ce39c70378e4430b296865cd962b --- [默认安装](install.md)无需任何选项。使用本页可以在现有反向代理后运行，或者在无法访问互联网时进行安装。 @@ -18,7 +18,7 @@ source_hash: 4b847976f3b3bc4fe4afe7b7946c5815df9fecf75dc39648621814e793288f9e ## Docker Compose 与托管平台 {#docker-compose-and-hosting-platforms} -在 Linux amd64 上使用发行版中的 `compose.yaml` 和 Docker Compose 2.26 或更高版本。发行流程会把节点元数据渲染进 [Compose 模板](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml)。Core 和 Web 使用 `latest` 镜像，PostgreSQL 使用 `postgres:16-alpine`。它会启动 PostgreSQL、Core 和 Web。Web 把 `/v1` 和 `/api/v1` 转发到 Core。数据通过目录 bind mount 挂载。一次性初始化服务会在该目录中生成随机机密信息并准备节点安装程序；Core 启动时执行数据库迁移。[Compose 配置](../configuration.md#compose-installations)负责管理各项设置和数据目录。 +在 Linux amd64 上使用发行版中的 `compose.yaml` 和 Docker Compose 2.26 或更高版本。发行流程会在 [Compose 模板](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml)中固定初始化镜像及源码版本。Core 和 Web 使用 `latest` 镜像，PostgreSQL 使用 `postgres:16-alpine`。它会启动 PostgreSQL、Core 和 Web。Web 把 `/v1` 和 `/api/v1` 转发到 Core。数据通过目录 bind mount 挂载。一次性初始化服务会在该目录中生成随机机密信息并准备节点安装程序；Core 启动时执行数据库迁移。[Compose 配置](../configuration.md#compose-installations)负责管理各项设置和数据目录。 进行本地试用时，请将同一发行版的 `compose.yaml` 和 `ports.yaml` 下载到同一个目录，然后运行： @@ -29,7 +29,7 @@ docker compose -f compose.yaml exec web oac-web core-key `oac-web core-key` 会将生成的 Core 密钥打印到终端，而不会将其写入容器日志。打开 `http://localhost:8080` 并使用该密钥登录。所有安装机密信息都会自动生成；重启时请保留同一个 Compose 项目及其数据目录。 -首次初始化会下载并验证该发布版本中约 385 MB 的控制归档文件，仅保留较小的节点安装元数据。后续启动会验证已保存的文件，而不会再次下载。镜像需要额外下载。首次初始化中断后可以重新运行；如果现有数据库缺少安装机密信息，初始化会被拒绝。 +初始化镜像除初始化命令外，仅包含较小的节点安装元数据。首次启动会验证并复制这些元数据，无需下载控制归档或访问 GitHub Releases。后续启动会验证已保存的文件。容器镜像仍需拉取。首次初始化中断后可以重新运行；如果现有数据库缺少安装机密信息，初始化会被拒绝。 你可以在选择域名前进行部署：将 `OAC_PUBLIC_URL` 保持未设置或留空，然后在平台的域名准备好后，按照 [Compose 配置](../configuration.md#compose-installations)进行设置并重新部署。初始的 localhost 源地址允许服务启动；Web 仅接受已配置的主机，因此重新部署后即可通过平台域名访问。 diff --git a/docs/zh/maintainers.md b/docs/zh/maintainers.md index e5030fc3..efe6873f 100644 --- a/docs/zh/maintainers.md +++ b/docs/zh/maintainers.md @@ -1,7 +1,7 @@ --- title: "构建并发布 OpenAgentCore" source: docs/maintainers.md -source_hash: ed4a16eca4710a2bc8be67b07d465e599d77638cb53ef34059fe4635671f18a0 +source_hash: e49645e98eac4e584dabd59b2b90c4e89b0c0fd521b5e1be9829ad3286447bd6 --- 本指南面向负责构建和发布 OpenAgentCore 的维护者。要安装 Core 和 Web，请使用 [安装指南](getting-started/install.md)。安装器代码遵循的规则见 [部署](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/README.md) 和 [节点安装器](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/node/README.md)；必需检查见 [CONTRIBUTING](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/CONTRIBUTING.md#required-checks)。 @@ -37,6 +37,8 @@ make build-core-distribution 构建过程会复用 Core、Web、Runtime、SDK 和辅助程序构建器。清单会记录提交和源代码树、镜像配置及 OCI 清单摘要、Runtime OCI 清单摘要、microsandbox 运行时和固件哈希，以及每个 Runtime 和节点构件的大小与 SHA-256；原生安装器在[目录](#native-installers)中仅记录其 SHA-256。输出包括控制归档及其 `.sha256`、可选的离线归档，以及带版本号的 Runtime、节点和原生安装器资源。此过程不会发布任何内容。如果目标目录中已包含此提交的分发包，重建会拒绝执行。 +ingress 镜像仅包含 `oac` 和四个节点元数据文件：`manifest.json`、`SHA256SUMS`、`node-install.pyz` 与 `runtime/seccomp.json`。构建流程根据与分发包相同的 Runtime 和构件身份生成其清单，再禁用网络验证初始化。节点清单不包含控制平面镜像身份，因此可在 ingress 镜像生成前打包。 + 控制归档不包含 Runtime 镜像或节点执行构件；离线归档包含这些内容。[下载契约](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/node/README.md#download-contract)说明了节点如何获取这些内容。 分发包会携带 `scripts/core-distribution-manifest.py` 中 `BUNDLED_DOCS` 列出的文档。随包文档之间的链接保持相对路径；其他所有相对链接都会重写为该捆绑包对应提交在 GitHub 上的同一文件。链接或锚点无法解析时，构建会失败；`make check-distribution` 会对 `example/` 之外每个受 Git 跟踪的 Markdown 文件运行相同检查。添加或移动安装器或其输出所引用的文档时，请更新该列表。 @@ -132,7 +134,7 @@ git push origin v1.2.3 ### 容器注册表 {#container-registry} -版本发布和手动的 `build-` 草稿都会将 Linux amd64 镜像发布为 `ghcr.io/minimax-ai/openagentcore/:`，其中 `` 为 `core`、`web`、`runtime` 或 `ingress`。例如，`ghcr.io/minimax-ai/openagentcore/core:v1.2.3`。草稿使用标签 `build-`。PostgreSQL 使用其上游镜像，不会重新发布。注册表镜像从发布归档中加载，不会重新构建。仅当现有版本标签的镜像配置摘要与本次发布相同时才复用该标签；如果镜像不同，则停止发布。稳定版还会把每个组件的 `latest` 标签移到该镜像。预发布和草稿不会改动 `latest`。SemVer 构建元数据在容器标签中使用 `_` 代替 `+`；长度超过 128 个字符的版本字符串无法发布到 GHCR。镜像验证之后，发布器会上传为该发行版渲染的 `compose.yaml` 和 `ports.yaml` 及其校验和。草稿 Release 保持未发布。 +版本发布和手动的 `build-` 草稿都会将 Linux amd64 镜像发布为 `ghcr.io/minimax-ai/openagentcore/:`，其中 `` 为 `core`、`web`、`runtime` 或 `ingress`。例如，`ghcr.io/minimax-ai/openagentcore/core:v1.2.3`。草稿使用标签 `build-`。PostgreSQL 使用其上游镜像，不会重新发布。注册表镜像从发布归档中加载，不会重新构建。仅当现有版本标签的镜像配置摘要与本次发布相同时才复用该标签；如果镜像不同，则停止发布。稳定版还会把每个组件的 `latest` 标签移到该镜像。预发布和草稿不会改动 `latest`。SemVer 构建元数据在容器标签中使用 `_` 代替 `+`；长度超过 128 个字符的版本字符串无法发布到 GHCR。镜像验证之后，发布器会上传为该发行版渲染的 `compose.yaml` 和 `ports.yaml` 及其校验和。Compose 使用注册表摘要固定 ingress 镜像；如果镜像构建版本与 Compose 版本不同，初始化会拒绝运行。草稿 Release 保持未发布。 合并的构建/发布作业使用具有 `packages: write` 权限的 `GITHUB_TOKEN`。首次发布时，GitHub 会将每个容器软件包创建为私有：软件包管理员必须先在各自的软件包设置中将全部四个软件包改为 **Public**，用户才能匿名拉取。请参阅 [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry)。更改可见性后，请验证未认证拉取。仅更改仓库可见性并不会使新的容器软件包变为公开。 @@ -179,7 +181,7 @@ gh workflow run core-release --repo MiniMax-AI/OpenAgentCore --ref main \ `.github/actionlint.yaml` 会选择 hygiene 和 lint。已知工作流变更会选择其使用方：CI review 和 actionlint 工作流运行 hygiene 和 lint；原生工作流变更会添加原生检查；API 验收工作流变更会添加启用容器验收的 API 检查；网站工作流变更会添加网站检查。共享 Node 操作会选择使用它的每个作业以及 lint。新工作流或未分类的工作流/操作会选择完整门禁，直至在计划器中声明其使用方。计划器测试和 CI 测量脚本运行 hygiene；更改计划器本身会运行完整门禁。 -Compose 模板和 Compose 测试发生变更时，会同时选择 `distribution` 固定数据和 `compose` 冒烟作业；Core、Web、共享 Go 软件包和镜像 Dockerfile 的变更也会选择冒烟作业。安装 Docker 后，可在本地运行 `python3 scripts/compose-smoke.py` 重复该测试。该脚本使用唯一的项目、自动分配的回环端口，并将在 `~/.oac/tests/` 下生成构件；退出时移除其容器和数据卷。CI 还会在冒烟步骤失败或中断后执行清理。诊断信息会显示容器状态，但不会打印 HTTP 响应正文或登录密钥。Core、Web 和 ingress 镜像都从当前检出构建；Web 提供占位页面而不是控制台构建。节点元数据来自 `deploy/compose/smoke-pins.json` 固定的发布版本。该测试检查通用 Compose 行为；它不会运行 Dokploy/Coolify 实例，也不会执行模型。 +Compose 模板和 Compose 测试发生变更时，会同时选择 `distribution` 固定数据和 `compose` 冒烟作业；Core、Web、共享 Go 软件包和镜像 Dockerfile 的变更也会选择冒烟作业。安装 Docker 后，可在本地运行 `python3 scripts/compose-smoke.py` 重复该测试。该脚本使用唯一的项目、自动分配的回环端口，并将在 `~/.oac/tests/` 下生成构件；退出时移除其容器和数据卷。CI 还会在冒烟步骤失败或中断后执行清理。诊断信息会显示容器状态，但不会打印 HTTP 响应正文或登录密钥。Core、Web 和 ingress 镜像都从当前检出构建；Web 提供占位页面而不是控制台构建。构建时的节点元数据来自 `deploy/compose/smoke-pins.json` 固定的发布版本；初始化容器禁用网络运行。该测试检查通用 Compose 行为；它不会运行 Dokploy/Coolify 实例，也不会执行模型。 Go 模块和工作区输入会选择后端、API（包括容器）、原生和分发检查。每个 Node 模块都拥有自己的清单和锁文件。网站依赖项会选择网站检查；Web 依赖项会选择 Web 和浏览器检查；示例依赖项会选择示例检查；共享 TypeScript 客户端依赖项会选择 Web、浏览器和示例检查；Claude 适配器依赖项会选择 Harness、原生和分发检查。共享包管理器配置会选择所有 Node 使用方。根 TypeScript 配置会选择 Web 和示例检查；适配器 TypeScript 配置会选择 Harness 和原生检查。每个所选集合都包含 hygiene。混合变更会累加其使用方，并且每个作业都读取同一计划，而不是维护各自的路径列表。例如，仅修改通知的 PR 会跳过数据库、浏览器和原生作业，而同时修改通知和 Core 的 PR 会添加后端和 API 检查。 diff --git a/scripts/build-core-distribution.sh b/scripts/build-core-distribution.sh index 2024eb2e..bc661b25 100755 --- a/scripts/build-core-distribution.sh +++ b/scripts/build-core-distribution.sh @@ -149,11 +149,6 @@ cp -R apps/web/dist "$stage/web/dist" cp services/web/Dockerfile "$stage/web/Dockerfile" build_image web "$stage/web" -mkdir -p "$stage/ingress" -cp deploy/distribution/Ingress.Dockerfile "$stage/ingress/Dockerfile" -cp "$stage/core/bin/oac" "$stage/ingress/oac" -build_image ingress "$stage/ingress" - CGO_ENABLED=0 go build -mod=readonly -trimpath -o "$stage/oac-daemon" ./apps/daemon/cmd/oac-daemon cp "$stage/oac-daemon" "$bundle/native/bin/oac-daemon" codex_image="${CORE_DISTRIBUTION_CODEX_IMAGE:-}" @@ -205,7 +200,7 @@ fi docker image inspect --format '{{.Id}}' "$database_image" > "$stage/database.id" docker run --rm --network none --entrypoint postgres "$(cat "$stage/database.id")" --version \ | python3 -c 'import sys; value=sys.stdin.read(); assert value.startswith("postgres (PostgreSQL) 16."), "Distribution requires PostgreSQL 16"' -for name in core web runtime database ingress; do +for name in core web runtime database; do image="$(cat "$stage/$name.id")" python3 scripts/core-distribution-manifest.py verify-image "$image" docker image save --output "$bundle/images/$name.tar" "$image" @@ -223,7 +218,17 @@ msb=(docker run --rm --network none --user "$(id -u):$(id -g)" \ --entrypoint /opt/microsandbox/msb "$core_image") "${msb[@]}" image load --input /runtime.tar --tag oac-runtime:distribution --quiet "${msb[@]}" image inspect oac-runtime:distribution --format json > "$stage/runtime-inspect.json" -python3 scripts/core-distribution-manifest.py manifest "$bundle" "$stage" "$revision" "$source_tree" "$release_base_url" "$offline" +python3 scripts/core-distribution-manifest.py node-payload "$bundle" "$stage" "$revision" "$source_tree" "$release_base_url" "$offline" +mkdir -p "$stage/ingress" +cp deploy/distribution/Ingress.Dockerfile "$stage/ingress/Dockerfile" +cp "$stage/core/bin/oac" "$stage/ingress/oac" +build_image ingress "$stage/ingress" + +docker run --rm --network none --entrypoint /usr/local/bin/oac \ + --env "OAC_REVISION=$revision" --tmpfs /data "$(cat "$stage/ingress.id")" init +python3 scripts/core-distribution-manifest.py verify-image "$(cat "$stage/ingress.id")" +docker image save --output "$bundle/images/ingress.tar" "$(cat "$stage/ingress.id")" +python3 scripts/core-distribution-manifest.py manifest "$bundle" "$stage" require_clean_source if [[ "$(git -C "$repo_root" rev-parse HEAD)" != "$revision" ]]; then printf 'Source changed during distribution build\n' >&2 diff --git a/scripts/compose-smoke.py b/scripts/compose-smoke.py index 72fcf4d8..4d599250 100644 --- a/scripts/compose-smoke.py +++ b/scripts/compose-smoke.py @@ -14,6 +14,7 @@ from pathlib import Path import re import signal +import tarfile import subprocess import tempfile import urllib.error @@ -26,14 +27,39 @@ render_spec.loader.exec_module(render_compose) +def prepare_pinned_payload(destination): + """Build fixtures use verified release metadata; initialization stays offline.""" + pins = json.loads((ROOT / 'deploy/compose/smoke-pins.json').read_text()) + stem = 'oac-' + pins['revision'] + '-linux-amd64' + destination.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=destination.parent) as temporary: + archive = Path(temporary) / 'release.tar.gz' + urllib.request.urlretrieve(pins['release_base'] + stem + '.tar.gz', archive) + digest = hashlib.sha256() + with archive.open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(block) + if digest.hexdigest() != pins['archive_checksum']: + raise ValueError('Pinned smoke metadata checksum mismatch') + with tarfile.open(archive, 'r:gz') as release: + for name in ('manifest.json', 'SHA256SUMS', 'node-install.pyz', 'runtime/seccomp.json'): + member = release.getmember(stem + '/' + name) + if not member.isfile() or member.size > 1024 * 1024: + raise ValueError('Invalid smoke metadata member') + target = destination / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(release.extractfile(member).read()) + return pins['revision'] + + def build_images(directory, tag): revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() protocol = re.search(r'const Version = "([^"]+)"', (ROOT / 'internal/agentdaemon/proto/version.go').read_text()).group(1) go_env = {**os.environ, 'CGO_ENABLED': '0', 'GOOS': 'linux', 'GOARCH': 'amd64'} - def go_build(package, output): + def go_build(package, output, build_revision=revision): output.parent.mkdir(parents=True, exist_ok=True) - subprocess.run(['go', 'build', '-trimpath', '-ldflags', '-X main.buildRevision=' + revision, + subprocess.run(['go', 'build', '-trimpath', '-ldflags', '-X main.buildRevision=' + build_revision, '-o', str(output), './' + package], cwd=ROOT, env=go_env, check=True) contexts = {name: directory / ('image-' + name) for name in ('core', 'web', 'ingress')} @@ -54,7 +80,8 @@ def go_build(package, output): (web / 'dist/index.html').write_text('Compose smoke\n') (web / 'Dockerfile').write_bytes((ROOT / 'services/web/Dockerfile').read_bytes()) ingress = contexts['ingress'] - go_build('services/core/cmd/oac', ingress / 'oac') + payload_revision = prepare_pinned_payload(ingress / 'node-payload') + go_build('services/core/cmd/oac', ingress / 'oac', payload_revision) (ingress / 'Dockerfile').write_bytes((ROOT / 'deploy/distribution/Ingress.Dockerfile').read_bytes()) for path in directory.glob('image-*/**/*'): path.chmod(0o755 if path.is_dir() or os.access(path, os.X_OK) else 0o644) @@ -79,14 +106,14 @@ def main(): pins = json.loads((ROOT / 'deploy/compose/smoke-pins.json').read_text()) rendered = directory / 'compose.yaml' rendered.write_text(render_compose.render({ - 'REVISION': pins['revision'], 'RELEASE_BASE': pins['release_base'], - 'ARCHIVE_CHECKSUM': pins['archive_checksum'], + 'REVISION': pins['revision'], + 'INIT_IMAGE': 'ghcr.io/minimax-ai/openagentcore/ingress@sha256:' + '0' * 64, })) override = directory / 'ports.json' images = build_images(directory, project.removeprefix('oac-smoke-')) def publish(port): - override.write_text(json.dumps({'services': {'web': {'ports': [ + override.write_text(json.dumps({'services': {'init': {'network_mode': 'none'}, 'web': {'ports': [ {'target': 8080, 'published': str(port), 'host_ip': '127.0.0.1'}, ]}}})) @@ -188,7 +215,7 @@ def terminate(_signum, _frame): assert updated['public_url'] == origin, 'The new public URL did not take effect' assert any(p['id'] == project_data['id'] for p in get('/core/v1/projects')['data']), 'Project was lost' assert get('/v1/files/' + uploaded['id'], headers=api)['bytes'] == len(content), 'Uploaded file metadata was lost' - assert 'Downloading and verifying' not in private_logs(key, project_key), 'Completed initialization downloaded again' + assert 'Bundled node installation metadata verified' not in private_logs(key, project_key), 'Completed initialization recopied metadata' print('PASS: startup, origin validation, sign-in, API, upload, node installer and persistent installation', flush=True) except BaseException: # Service status identifies failed containers without dumping secret-bearing logs. diff --git a/scripts/core-distribution-manifest.py b/scripts/core-distribution-manifest.py index df051806..c00f363b 100644 --- a/scripts/core-distribution-manifest.py +++ b/scripts/core-distribution-manifest.py @@ -305,7 +305,7 @@ def bootstraps(bundle, epoch, revision): zipapp.create_archive(directory, bundle / "node-install.pyz", compressed=True) -def manifest(bundle, stage, revision, source_tree, artifact_base_url="", offline="0"): +def node_payload(bundle, stage, revision, source_tree, artifact_base_url="", offline="0"): bundle, stage = pathlib.Path(bundle), pathlib.Path(stage) artifact_base_url = release_base(artifact_base_url) if not artifact_base_url and offline != "1": @@ -320,7 +320,7 @@ def manifest(bundle, stage, revision, source_tree, artifact_base_url="", offline raise ValueError("msb imported an unexpected Runtime platform") identities = {name: image_identities(bundle / "images" / (name + ".tar"), (stage / (name + ".id")).read_text().strip()) - for name in ("core", "web", "runtime", "database", "ingress")} + for name in ("runtime",)} metadata = { "source_commit": revision, "source_tree": source_tree, @@ -336,6 +336,24 @@ def manifest(bundle, stage, revision, source_tree, artifact_base_url="", offline "firmware_sha256": sha256(stage / "core/microsandbox/libkrunfw.so.5.6.1"), }, } + payload = stage / "ingress/node-payload" + payload.mkdir(parents=True) + (payload / "manifest.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") + for name in ("node-install.pyz", "runtime/seccomp.json"): + target = payload / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(bundle / name, target) + checksums(payload) + + +def manifest(bundle, stage): + bundle, stage = pathlib.Path(bundle), pathlib.Path(stage) + metadata = json.loads((stage / "ingress/node-payload/manifest.json").read_text()) + for name in ("core", "web", "database", "ingress"): + config, digest = image_identities(bundle / "images" / (name + ".tar"), + (stage / (name + ".id")).read_text().strip()) + metadata["images"][name] = config + metadata["image_manifest_digests"][name] = digest (bundle / "manifest.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") checksums(bundle) @@ -566,7 +584,7 @@ def check(image, link): if __name__ == "__main__": commands = {"extract-runtime": extract_runtime, "verify-runtime": verify_runtime, "verify-image": verify_image, - "built-image": built_image, "manifest": manifest, "archive": archive, "bootstraps": bootstraps, + "built-image": built_image, "node-payload": node_payload, "manifest": manifest, "archive": archive, "bootstraps": bootstraps, "release-base": release_base, "docs": docs, "native-catalog": native_catalog, "native-offline": native_offline} try: commands[sys.argv[1]](*sys.argv[2:]) diff --git a/scripts/core-distribution-manifest.test.py b/scripts/core-distribution-manifest.test.py index 8c6576a2..4719f05a 100644 --- a/scripts/core-distribution-manifest.test.py +++ b/scripts/core-distribution-manifest.test.py @@ -108,14 +108,30 @@ def setUp(self): for name in ("core", "web", "runtime", "database", "ingress"): self.identities[name] = image_archive(self.bundle / "images" / (name + ".tar"), name) (self.stage / (name + ".id")).write_text(self.identities[name][0] + "\n") + (self.bundle / "node-install.pyz").write_bytes(b"node installer") self.runtime_bytes = (self.bundle / "images/runtime.tar").read_bytes() def manifest(self, base=RELEASE_BASE, offline="0"): - distribution.manifest(self.bundle, self.stage, REVISION, TREE, base, offline) + distribution.node_payload(self.bundle, self.stage, REVISION, TREE, base, offline) + distribution.manifest(self.bundle, self.stage) def write_inspection(self): (self.stage / "runtime-inspect.json").write_text(json.dumps(self.inspection)) + def test_init_payload_contains_only_verified_node_metadata(self): + self.manifest() + payload = self.stage / "ingress/node-payload" + files = sorted(p.relative_to(payload).as_posix() for p in payload.rglob("*") if p.is_file()) + self.assertEqual(files, ["SHA256SUMS", "manifest.json", "node-install.pyz", "runtime/seccomp.json"]) + sums = dict(line.split(" ", 1)[::-1] for line in (payload / "SHA256SUMS").read_text().splitlines()) + for name, checksum in sums.items(): + self.assertEqual(distribution.sha256(payload / name), checksum) + bundled = json.loads((payload / "manifest.json").read_text()) + full = json.loads((self.bundle / "manifest.json").read_text()) + for name in ("source_commit", "platform", "artifacts", "runtime_ref", "microsandbox"): + self.assertEqual(bundled[name], full[name]) + self.assertEqual(list(bundled["images"]), ["runtime"]) + def test_oci_manifest_identity_is_distinct_from_docker_config_identity(self): self.manifest() metadata = json.loads((self.bundle / "manifest.json").read_text()) diff --git a/scripts/publish-core-release.py b/scripts/publish-core-release.py index 9ef65467..7f27e030 100644 --- a/scripts/publish-core-release.py +++ b/scripts/publish-core-release.py @@ -274,8 +274,7 @@ def upload(path): images = publish_images(assets, repository, revision, tag, floating_latest=mode == "publish" and stable) compose_files = render_compose.write_assets(assets, { "REVISION": revision, - "RELEASE_BASE": "https://github.com/" + repository + "/releases/download/" + tag + "/", - "ARCHIVE_CHECKSUM": distribution.sha256(assets / (stem + ".tar.gz")), + "INIT_IMAGE": images["ingress"]["digest"], }) expected.update({path.name: path.stat().st_size for path in compose_files}) parallel_each(upload, compose_files) diff --git a/scripts/render-compose.py b/scripts/render-compose.py index 50044cbb..6e2b4871 100644 --- a/scripts/render-compose.py +++ b/scripts/render-compose.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 """Fill the Compose template with one release's node metadata. -The template is deploy/compose/compose.yaml. Images stay on their default -latest tags. A release publishes the rendered file; this script does not run Docker. +The template is deploy/compose/compose.yaml. The initialization image is pinned; Core and Web default to latest. A release publishes the rendered file; this script does not run Docker. """ import hashlib import pathlib @@ -12,7 +11,7 @@ ROOT = pathlib.Path(__file__).resolve().parents[1] TEMPLATE = ROOT / "deploy/compose/compose.yaml" PORTS = ROOT / "deploy/compose/ports.yaml" -TOKENS = ("REVISION", "RELEASE_BASE", "ARCHIVE_CHECKSUM") +TOKENS = ("REVISION", "INIT_IMAGE") def render(values): @@ -22,11 +21,8 @@ def render(values): raise ValueError("Missing Compose values: " + ", ".join(missing)) if not re.fullmatch(r"[0-9a-f]{40}", values["REVISION"]): raise ValueError("REVISION must be a full source commit SHA") - if not re.fullmatch(r"[0-9a-f]{64}", values["ARCHIVE_CHECKSUM"]): - raise ValueError("ARCHIVE_CHECKSUM must be a SHA-256 hex digest") - base = values["RELEASE_BASE"] - if not base.startswith("https://") or not base.endswith("/") or " " in base: - raise ValueError("RELEASE_BASE must be an https URL ending with /") + if not re.fullmatch(r"ghcr\.io/[a-z0-9._/-]+@sha256:[0-9a-f]{64}", values["INIT_IMAGE"]): + raise ValueError("INIT_IMAGE must be an immutable GHCR image reference") text = TEMPLATE.read_text() for name in TOKENS: token = "__OAC_" + name + "__" diff --git a/services/core/cmd/oac/init.go b/services/core/cmd/oac/init.go index bb5a5405..63a8ca93 100644 --- a/services/core/cmd/oac/init.go +++ b/services/core/cmd/oac/init.go @@ -1,9 +1,6 @@ package main import ( - "archive/tar" - "compress/gzip" - "context" "crypto/rand" "crypto/sha256" "encoding/base64" @@ -11,19 +8,16 @@ import ( "encoding/json" "errors" "fmt" - "io" "io/fs" - "net/http" "os" "path/filepath" + "strings" "syscall" - "time" "github.com/google/uuid" ) -// releaseMembers are the archive files a Compose installation keeps for Web's -// node payload; the rest of the release archive is only checksummed. +// releaseMembers are the only files copied from the initialization image. var releaseMembers = []string{"manifest.json", "SHA256SUMS", "node-install.pyz", "runtime/seccomp.json"} var dataOwners = []struct { @@ -34,86 +28,56 @@ var dataOwners = []struct { var chown = os.Chown type releaseIdentity struct { - revision, base, checksum string + revision string } -func releaseFromEnv() (releaseIdentity, error) { - identity := releaseIdentity{os.Getenv("OAC_REVISION"), os.Getenv("OAC_RELEASE_BASE"), os.Getenv("OAC_ARCHIVE_CHECKSUM")} - for name, value := range map[string]string{"OAC_REVISION": identity.revision, "OAC_RELEASE_BASE": identity.base, "OAC_ARCHIVE_CHECKSUM": identity.checksum} { - if value == "" { - return identity, errors.New(name + " is required") - } +func initCommand() error { + revision := os.Getenv("OAC_REVISION") + if revision == "" { + return errors.New("OAC_REVISION is required") } - return identity, nil -} - -func initCommand(ctx context.Context) error { - release, err := releaseFromEnv() - if err != nil { - return err + if revision != buildRevision { + return errors.New("initialization image does not match the Compose release") } syscall.Umask(0o077) - return initialize("/data", release, func() (map[string][]byte, error) { return downloadRelease(ctx, release) }) -} - -func downloadRelease(ctx context.Context, release releaseIdentity) (map[string][]byte, error) { - fmt.Println("Downloading and verifying the matched node installation metadata") - archive := "oac-" + release.revision + "-linux-amd64" - ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) - defer cancel() - request, err := http.NewRequestWithContext(ctx, http.MethodGet, release.base+archive+".tar.gz", nil) - if err != nil { - return nil, err - } - response, err := http.DefaultClient.Do(request) - if err != nil { - return nil, err - } - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("release metadata download returned HTTP %d", response.StatusCode) - } - return readRelease(response.Body, release) + release := releaseIdentity{revision} + return initialize("/data", release, func() (map[string][]byte, error) { + return readRelease("/opt/oac/node-payload", release) + }) } -func readRelease(body io.Reader, release releaseIdentity) (map[string][]byte, error) { - archive := "oac-" + release.revision + "-linux-amd64/" - hash := sha256.New() - stream := io.TeeReader(body, hash) - compressed, err := gzip.NewReader(stream) - if err != nil { - return nil, err - } +func readRelease(root string, release releaseIdentity) (map[string][]byte, error) { files := map[string][]byte{} - wanted := map[string]bool{} for _, name := range releaseMembers { - wanted[archive+name] = true - } - entries := tar.NewReader(compressed) - for { - header, err := entries.Next() - if errors.Is(err, io.EOF) { - break - } + path := filepath.Join(root, name) + info, err := os.Lstat(path) if err != nil { return nil, err } - if !wanted[header.Name] { - continue - } - name := header.Name[len(archive):] - if _, seen := files[name]; seen || header.Typeflag != tar.TypeReg || header.Size > 1<<20 { + if !info.Mode().IsRegular() || info.Size() > 1<<20 { return nil, errors.New("invalid release metadata member") } - if files[name], err = io.ReadAll(entries); err != nil { + files[name], err = os.ReadFile(path) + if err != nil { return nil, err } } - if _, err := io.Copy(io.Discard, stream); err != nil { - return nil, err + sums := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(string(files["SHA256SUMS"])), "\n") { + checksum, name, ok := strings.Cut(line, " ") + if !ok || sums[name] != "" { + return nil, errors.New("invalid release metadata checksums") + } + sums[name] = checksum } - if hex.EncodeToString(hash.Sum(nil)) != release.checksum || len(files) != len(releaseMembers) { - return nil, errors.New("release metadata checksum mismatch") + for name, data := range files { + if name == "SHA256SUMS" { + continue + } + sum := sha256.Sum256(data) + if hex.EncodeToString(sum[:]) != sums[name] { + return nil, errors.New("release metadata checksum mismatch") + } } var manifest struct { SourceCommit string `json:"source_commit"` @@ -125,6 +89,7 @@ func readRelease(body io.Reader, release releaseIdentity) (map[string][]byte, er if manifest.SourceCommit != release.revision || manifest.Platform != "linux/amd64" { return nil, errors.New("release identity mismatch") } + fmt.Println("Bundled node installation metadata verified") return files, nil } diff --git a/services/core/cmd/oac/init_test.go b/services/core/cmd/oac/init_test.go index c4b89db7..3c26735d 100644 --- a/services/core/cmd/oac/init_test.go +++ b/services/core/cmd/oac/init_test.go @@ -1,9 +1,7 @@ package main import ( - "archive/tar" "bytes" - "compress/gzip" "crypto/sha256" "encoding/base64" "encoding/hex" @@ -23,7 +21,7 @@ func initFixture(t *testing.T) (string, releaseIdentity, map[string][]byte) { previous := chown chown = func(string, int, int) error { return nil } t.Cleanup(func() { chown = previous }) - release := releaseIdentity{strings.Repeat("d", 40), "https://example.com/releases/v1/", strings.Repeat("e", 64)} + release := releaseIdentity{strings.Repeat("d", 40)} files := map[string][]byte{} for _, name := range releaseMembers { files[name] = []byte("fixture") @@ -145,49 +143,62 @@ func TestInterruptedInitializationKeepsGeneratedKeys(t *testing.T) { } } -func releaseArchive(t *testing.T, release releaseIdentity, files map[string][]byte) []byte { +func metadataDirectory(t *testing.T, files map[string][]byte) string { t.Helper() - var output bytes.Buffer - compressed := gzip.NewWriter(&output) - archive := tar.NewWriter(compressed) - add := func(name string, data []byte) { - if err := archive.WriteHeader(&tar.Header{Name: "oac-" + release.revision + "-linux-amd64/" + name, Mode: 0o644, Size: int64(len(data)), Typeflag: tar.TypeReg}); err != nil { - t.Fatal(err) + root := t.TempDir() + sums := "" + for _, name := range releaseMembers { + if name == "SHA256SUMS" { + continue } - _, _ = archive.Write(data) + sum := sha256.Sum256(files[name]) + sums += hex.EncodeToString(sum[:]) + " " + name + "\n" } for name, data := range files { - add(name, data) + if name == "SHA256SUMS" { + data = []byte(sums) + } + path := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } } - add("images/core.tar", bytes.Repeat([]byte("ignored image data"), 1000)) - _ = archive.Close() - _ = compressed.Close() - return output.Bytes() + return root } -func TestReadReleaseVerifiesTheWholeStreamAndKeepsOnlyMetadata(t *testing.T) { +func TestReadBundledReleaseRejectsCorruptMissingAndForeignMetadata(t *testing.T) { _, release, files := initFixture(t) - data := releaseArchive(t, release, files) - sum := sha256.Sum256(data) - release.checksum = hex.EncodeToString(sum[:]) - got, err := readRelease(bytes.NewReader(data), release) - if err != nil || len(got) != len(files) { - t.Fatalf("got %d files, %v", len(got), err) - } - for name, want := range files { - if !bytes.Equal(got[name], want) { - t.Fatalf("%s differs", name) - } + root := metadataDirectory(t, files) + got, err := readRelease(root, release) + if err != nil || len(got) != len(releaseMembers) { + t.Fatalf("files = %v, err = %v", got, err) + } + path := filepath.Join(root, "node-install.pyz") + if err := os.WriteFile(path, []byte("corrupt"), 0o600); err != nil { + t.Fatal(err) } - release.checksum = strings.Repeat("0", 64) - if _, err := readRelease(bytes.NewReader(data), release); err == nil || !strings.Contains(err.Error(), "checksum mismatch") { - t.Fatalf("bad checksum: %v", err) + if _, err := readRelease(root, release); err == nil || !strings.Contains(err.Error(), "checksum mismatch") { + t.Fatalf("corrupt: %v", err) } - delete(files, "node-install.pyz") - data = releaseArchive(t, release, files) - sum = sha256.Sum256(data) - release.checksum = hex.EncodeToString(sum[:]) - if _, err := readRelease(bytes.NewReader(data), release); err == nil || !strings.Contains(err.Error(), "checksum mismatch") { - t.Fatalf("missing member: %v", err) + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if _, err := readRelease(root, release); !os.IsNotExist(err) { + t.Fatalf("missing: %v", err) + } + root = metadataDirectory(t, files) + release.revision = strings.Repeat("f", 40) + if _, err := readRelease(root, release); err == nil || !strings.Contains(err.Error(), "identity mismatch") { + t.Fatalf("foreign: %v", err) + } +} + +func TestInitRejectsMismatchedImageBeforeTouchingData(t *testing.T) { + t.Setenv("OAC_REVISION", strings.Repeat("f", 40)) + if err := initCommand(); err == nil || !strings.Contains(err.Error(), "image does not match") { + t.Fatalf("err = %v", err) } } diff --git a/services/core/cmd/oac/main.go b/services/core/cmd/oac/main.go index e2159239..254993a5 100644 --- a/services/core/cmd/oac/main.go +++ b/services/core/cmd/oac/main.go @@ -38,7 +38,7 @@ func usage() { func run(ctx context.Context, command string, args []string) error { switch command { case "init": - return initCommand(ctx) + return initCommand() } root, err := installDir() if err != nil { From 794cfd8cd1f71418084d8c44159b18ee43818557 Mon Sep 17 00:00:00 2001 From: yuanhe Date: Sat, 3 Oct 2026 19:33:06 +0800 Subject: [PATCH 2/2] Publish Web from the single Compose file --- deploy/README.md | 4 ++-- deploy/compose/compose.yaml | 2 ++ deploy/compose/dokploy.toml | 2 +- deploy/compose/ports.yaml | 6 ------ deploy/compose/test_compose.py | 19 ++++++++++++------- deploy/install.dev.sh | 2 -- deploy/install.sh | 13 ++++--------- deploy/test_install.py | 3 +-- docs/configuration.md | 5 ++--- docs/getting-started/install-options.md | 10 +++++----- docs/maintainers.md | 2 +- docs/zh/configuration.md | 7 +++---- docs/zh/getting-started/install-options.md | 12 ++++++------ docs/zh/maintainers.md | 4 ++-- scripts/ci_plan_test.py | 2 +- scripts/compose-smoke.py | 15 +++++++-------- scripts/publish-core-release.test.py | 7 ++++--- scripts/render-compose.py | 4 +--- 18 files changed, 54 insertions(+), 65 deletions(-) delete mode 100644 deploy/compose/ports.yaml diff --git a/deploy/README.md b/deploy/README.md index a69bfa3f..200b6803 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -3,13 +3,13 @@ | Path | Contents | | --- | --- | | `install.sh` | Host installer published with each release | -| `compose/` | Compose template, port overlays and their tests | +| `compose/` | Compose template and its tests | | `distribution/` | Image Dockerfiles | | `node/` | [Node installer](node/README.md), packaged as `node-install.pyz` | ## Installation -`install.sh` downloads its release's `compose.yaml` and port files, checks them against `compose-sha256sums.txt`, writes `.env`, and starts Compose. Core applies database migrations when it starts. The host needs Linux amd64 and Docker Compose 2.26 or newer. [Configuration](../docs/configuration.md) owns the installation layout and settings. +`install.sh` downloads its release's `compose.yaml`, checks it against `compose-sha256sums.txt`, writes `.env`, and starts Compose. Core applies database migrations when it starts. The host needs Linux amd64 and Docker Compose 2.26 or newer. [Configuration](../docs/configuration.md) owns the installation layout and settings. `oac` is a Go command (`services/core/cmd/oac`) in the Core image and the ingress image. The host copy implements `apply`, `core-key` and `rotate-core-key`; `core-key --show` runs `oac-web core-key` in the Web container. Start, stop, logs and removal are `docker compose`. `apply` runs `oac-core check-config` before recreating services. The ingress image runs data initialization as `oac init`, verifies and copies its bundled node metadata without network access, and contains no Python. No service receives a Docker socket. diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml index 85240dec..21ecceea 100644 --- a/deploy/compose/compose.yaml +++ b/deploy/compose/compose.yaml @@ -83,6 +83,8 @@ services: target: /state web: + ports: + - "${OAC_HOST:-127.0.0.1}:${OAC_WEB_PORT:-8080}:8080" image: ${OAC_IMAGE_WEB:-ghcr.io/minimax-ai/openagentcore/web:latest} platform: linux/amd64 user: "65532:65532" diff --git a/deploy/compose/dokploy.toml b/deploy/compose/dokploy.toml index 4ba5f428..27e7d734 100644 --- a/deploy/compose/dokploy.toml +++ b/deploy/compose/dokploy.toml @@ -6,6 +6,6 @@ main_domain = "${domain}" env = ["OAC_PUBLIC_URL=https://${main_domain}"] [[config.domains]] -serviceName = "gateway" +serviceName = "web" port = 8080 host = "${main_domain}" diff --git a/deploy/compose/ports.yaml b/deploy/compose/ports.yaml deleted file mode 100644 index 7dff93ba..00000000 --- a/deploy/compose/ports.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Host installation publishes Web. Hosting platforms omit this file and route -# to web:8080 themselves. -services: - web: - ports: - - "${OAC_HOST:-127.0.0.1}:${OAC_WEB_PORT:-8080}:8080" diff --git a/deploy/compose/test_compose.py b/deploy/compose/test_compose.py index f838f0ba..11f666e1 100644 --- a/deploy/compose/test_compose.py +++ b/deploy/compose/test_compose.py @@ -30,6 +30,8 @@ class ComposeTests(unittest.TestCase): def render(cls, public_url=None): env = dict(os.environ) env.pop('OAC_PUBLIC_URL', None) + env.pop('OAC_HOST', None) + env.pop('OAC_WEB_PORT', None) for name in ('OAC_IMAGE_CORE', 'OAC_IMAGE_WEB', 'OAC_IMAGE_INGRESS'): env.pop(name, None) env['OAC_DATA_DIR'] = '/tmp/oac-compose-fixture' @@ -54,7 +56,8 @@ def test_compose_uses_private_services_and_ordered_initialization(self): self.assertEqual(sorted(services), ['core', 'database', 'init', 'web']) for service in services.values(): self.assertNotIn('build', service) - self.assertNotIn('ports', service) + if service is not services['web']: + self.assertNotIn('ports', service) self.assertTrue(service['image'].endswith(':latest') or service['image'] == 'postgres:16-alpine' or service['image'].endswith('@sha256:' + 'e' * 64)) for volume in service.get('volumes', []): self.assertNotIn('docker.sock', json.dumps(volume)) @@ -83,13 +86,15 @@ def test_public_url_can_be_configured_after_initial_startup(self): for service, spec in self.compose['services'].items()}) def test_host_ports_publish_only_web(self): - env = dict(os.environ, OAC_DATA_DIR='/tmp/oac-compose-fixture', OAC_HOST='0.0.0.0') - hosted = json.loads(subprocess.check_output( + def ports(config): + return {name: [(port.get('host_ip'), port['published']) for port in service.get('ports', [])] + for name, service in config['services'].items() if service.get('ports')} + self.assertEqual(ports(self.compose), {'web': [('127.0.0.1', '8080')]}) + env = dict(os.environ, OAC_DATA_DIR='/tmp/oac-compose-fixture', OAC_HOST='0.0.0.0', OAC_WEB_PORT='9080') + configured = json.loads(subprocess.check_output( ['docker', 'compose', '--env-file', os.devnull, '-f', str(self.compose_file), - '-f', str(ROOT / 'deploy/compose/ports.yaml'), 'config', '--format', 'json'], env=env)) - published = {name: [(port.get('host_ip'), port['published']) for port in service.get('ports', [])] - for name, service in hosted['services'].items() if service.get('ports')} - self.assertEqual(published, {'web': [('0.0.0.0', '8080')]}) + 'config', '--format', 'json'], env=env)) + self.assertEqual(ports(configured), {'web': [('0.0.0.0', '9080')]}) def test_platform_network_injection_keeps_the_file_valid(self): # Dokploy isolated deployments attach a project network to every service. diff --git a/deploy/install.dev.sh b/deploy/install.dev.sh index 31005d27..eada59aa 100755 --- a/deploy/install.dev.sh +++ b/deploy/install.dev.sh @@ -111,12 +111,10 @@ pins = json.loads((root / "deploy/compose/smoke-pins.json").read_text()) "INIT_IMAGE": "ghcr.io/minimax-ai/openagentcore/ingress@sha256:" + "0" * 64, })) PY -cp "$repo_root/deploy/compose/ports.yaml" "$install_dir/ports.yaml" umask 077 cat >"$install_dir/.env" <&2 || true docker compose down --remove-orphans # Containers own data/; remove it from a container as well. - if [[ -d data ]]; then docker compose run --rm --no-deps --entrypoint find init /data -mindepth 1 -delete; fi + if [[ -d data ]]; then docker compose run --rm --no-deps --volume "$install_dir/data:/data" --entrypoint find database /data -mindepth 1 -delete; fi ) >/dev/null 2>&1 || true rm -rf "$install_dir" fi @@ -110,18 +110,13 @@ if [[ -z "$public_url" ]]; then public_url="http://localhost:$web_port"; local_o mkdir -p "$install_dir" chmod 700 "$install_dir" -files=(compose.yaml ports.yaml) curl --fail --silent --show-error --location "$asset_base/compose-sha256sums.txt" --output "$install_dir/compose-sha256sums.txt" -for name in "${files[@]}"; do - curl --fail --silent --show-error --location "$asset_base/$name" --output "$install_dir/$name" -done -(cd "$install_dir" && sha256sum --check --ignore-missing --quiet compose-sha256sums.txt) +curl --fail --silent --show-error --location "$asset_base/compose.yaml" --output "$install_dir/compose.yaml" +(cd "$install_dir" && sha256sum --check --quiet compose-sha256sums.txt) -compose_file="$(IFS=:; echo "${files[*]}")" umask 077 { echo "COMPOSE_PROJECT_NAME=oac-$(od -An -N5 -tx1 /dev/urandom | tr -d ' \n')" - echo "COMPOSE_FILE=$compose_file" echo "OAC_INSTALL_DIR=$install_dir" echo "OAC_HOST=$host_address" echo "OAC_WEB_PORT=$web_port" diff --git a/deploy/test_install.py b/deploy/test_install.py index 3effba8f..6393b28f 100644 --- a/deploy/test_install.py +++ b/deploy/test_install.py @@ -79,9 +79,8 @@ def test_env_holds_only_the_installation_choices(self): completed, _ = self.install(root, "--public-url", "https://core.example") self.assertEqual(completed.returncode, 0, completed.stderr) env = dict(line.split("=", 1) for line in (root / "oac/.env").read_text().splitlines()) - self.assertEqual(env["COMPOSE_FILE"], "compose.yaml:ports.yaml") self.assertEqual(env["OAC_PUBLIC_URL"], "https://core.example") - self.assertEqual(sorted(env), ["COMPOSE_FILE", "COMPOSE_PROJECT_NAME", "OAC_HOST", "OAC_INSTALL_DIR", "OAC_PUBLIC_URL", "OAC_WEB_PORT"]) + self.assertEqual(sorted(env), ["COMPOSE_PROJECT_NAME", "OAC_HOST", "OAC_INSTALL_DIR", "OAC_PUBLIC_URL", "OAC_WEB_PORT"]) def test_help_does_not_need_docker(self): help_text = subprocess.run(["bash", str(INSTALL), "--help"], capture_output=True, text=True, check=True) diff --git a/docs/configuration.md b/docs/configuration.md index bb6bd330..f91a3556 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -44,9 +44,8 @@ To change it, point the reverse proxy at the new address first, then edit `OAC_P | Variable | Default | Meaning | | --- | --- | --- | | `OAC_PUBLIC_URL` | `http://localhost:8080` | Origin applications, nodes, sandboxes and self-hosted executors use. See [changing the public URL](#changing-the-public-url) | -| `OAC_HOST` | `127.0.0.1` | Address published by `ports.yaml`. `install.sh` sets `0.0.0.0` | +| `OAC_HOST` | `127.0.0.1` | Web bind address published by `compose.yaml`. `install.sh` sets `0.0.0.0` | | `OAC_WEB_PORT` | `8080` | Host port of Web | -| `COMPOSE_FILE` | `compose.yaml:ports.yaml` | The Compose files. `ports.yaml` publishes Web; hosting platforms omit it | | `OAC_LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error` | | `OAC_LOG_FORMAT` | `auto` | `auto`, `text` or `json` | | `OAC_LOG_ADD_SOURCE` | unset | `1` adds source locations | @@ -124,7 +123,7 @@ The installer creates the installation directory, `~/.oac/core` by default, with | Path | Content | Changed by | | --- | --- | --- | | `.env` | [Process settings](#process-settings-configjson). The file you edit | You, then `oac apply` | -| `compose.yaml`, `ports.yaml` | The release's service definition. Do not edit them | The release | +| `compose.yaml` | The release's service definition. Do not edit them | The release | | `oac` | The [management command](./getting-started/operations.md#the-oac-command), copied from the Core image | The installer | | `data/secrets/web/core.key` | The [Core key](./getting-started/operations.md#core-key) | `oac rotate-core-key` | | `data/secrets/core/credential.key` | Encryption key for what Core stores sealed in the database | Nothing. Keep it with the database | diff --git a/docs/getting-started/install-options.md b/docs/getting-started/install-options.md index 757ed171..41c4e50e 100644 --- a/docs/getting-started/install-options.md +++ b/docs/getting-started/install-options.md @@ -16,11 +16,11 @@ With the one-line command, append them after `bash -s --`. `--version TAG` selec Use the `compose.yaml` from a release with Docker Compose 2.26 or newer on Linux amd64. The release pins its initialization image and source revision in the [Compose template](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml). Core and Web use the `latest` images, and PostgreSQL uses `postgres:16-alpine`. It starts PostgreSQL, Core and Web. Web forwards `/v1` and `/api/v1` to Core. Data is bind-mounted from a directory. The one-time initialization service generates random secrets there and prepares the node installer; Core applies database migrations when it starts. [Compose configuration](../configuration.md#compose-installations) owns the settings and the data directory. -For a local trial, download `compose.yaml` and `ports.yaml` from the same release into one directory, then run: +For a local trial, download `compose.yaml` from a release into an empty directory, then run: ```sh -docker compose -f compose.yaml -f ports.yaml up -d --wait --wait-timeout 900 -docker compose -f compose.yaml exec web oac-web core-key +docker compose up -d --wait --wait-timeout 900 +docker compose exec web oac-web core-key ``` `oac-web core-key` prints the generated Core key to your terminal without writing it to container logs. Open `http://localhost:8080` and use that key to sign in. All installation secrets are generated automatically; keep the same Compose project and its data directory when restarting. @@ -31,11 +31,11 @@ You can deploy before choosing a domain: leave `OAC_PUBLIC_URL` unset or empty, ### Dokploy -Create a Docker Compose application and paste `compose.yaml`. Set `OAC_PUBLIC_URL` to the public HTTPS origin, enable isolated deployment, and add a domain for service `web`, port `8080`. Enable **HTTPS** and select a certificate provider such as **Let's Encrypt** for that domain before deploying. Deploy without `ports.yaml`; internal services publish no host ports. The [template metadata](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/dokploy.toml) supplies the generated domain and environment when packaging this Compose file for Dokploy's template catalog; HTTPS and its certificate provider still need to be enabled after import. +Create a Docker Compose application and paste `compose.yaml`. Set `OAC_PUBLIC_URL` to the public HTTPS origin, enable isolated deployment, and add a domain for service `web`, port `8080`. Enable **HTTPS** and select a certificate provider such as **Let's Encrypt** for that domain before deploying. Remove the `ports` block from service `web` before deploying so Dokploy routes to its container port without publishing a host port. The [template metadata](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/dokploy.toml) supplies the generated domain and environment when packaging this Compose file for Dokploy's template catalog; HTTPS and its certificate provider still need to be enabled after import. ### Coolify -Create a **Docker Compose Empty** service and paste `compose.yaml`. Set `OAC_PUBLIC_URL` to the public HTTPS origin and assign that domain to `web` on port `8080`. Add Coolify's `exclude_from_hc: true` to the `init` service definition so completed initialization does not affect its overall health. Save and deploy without `ports.yaml`; Coolify supplies HTTPS. +Create a **Docker Compose Empty** service and paste `compose.yaml`. Set `OAC_PUBLIC_URL` to the public HTTPS origin and assign that domain to `web` on port `8080`. Add Coolify's `exclude_from_hc: true` to the `init` service definition so completed initialization does not affect its overall health. Remove the `ports` block from service `web`, then save and deploy; Coolify routes to its container port and supplies HTTPS. On either platform, open its server terminal and run `docker compose ls` to find the deployed project name and Compose file. Using those exact values and the deployment's `OAC_PUBLIC_URL`, run `docker compose -p -f exec web oac-web core-key`, then sign in at the configured origin. The [Dokploy domain guide](https://docs.dokploy.com/docs/core/docker-compose/domains) and [Coolify Compose guide](https://coolify.io/docs/services/configuration/docker-compose) describe their domain and service controls. These are importable deployment files; no hosted marketplace listing is published by this repository. diff --git a/docs/maintainers.md b/docs/maintainers.md index 600a85ba..c7818da9 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -132,7 +132,7 @@ Distribution and Runtime archives use `pigz` level 6 with at most four compressi ### Container registry -Version releases and manual `build-` drafts publish Linux amd64 images as `ghcr.io/minimax-ai/openagentcore/:`, where `` is `core`, `web`, `runtime` or `ingress`. For example, `ghcr.io/minimax-ai/openagentcore/core:v1.2.3`. A draft uses the tag `build-`. PostgreSQL uses its upstream image and is not republished. The registry images are loaded from the release archives without rebuilding. Existing version tags are reused only when their image config digest matches the release; a different image stops publication. A stable release also moves each component's `latest` tag to that image. Prereleases and drafts leave `latest` unchanged. SemVer build metadata uses `_` in place of `+` in container tags; version strings longer than 128 characters cannot be published to GHCR. After the images are verified, the publisher uploads `compose.yaml` and `ports.yaml`, with checksums, rendered for that release. Compose pins the ingress image by its registry digest; initialization rejects an image whose build revision differs from the Compose revision. A draft Release stays unpublished. +Version releases and manual `build-` drafts publish Linux amd64 images as `ghcr.io/minimax-ai/openagentcore/:`, where `` is `core`, `web`, `runtime` or `ingress`. For example, `ghcr.io/minimax-ai/openagentcore/core:v1.2.3`. A draft uses the tag `build-`. PostgreSQL uses its upstream image and is not republished. The registry images are loaded from the release archives without rebuilding. Existing version tags are reused only when their image config digest matches the release; a different image stops publication. A stable release also moves each component's `latest` tag to that image. Prereleases and drafts leave `latest` unchanged. SemVer build metadata uses `_` in place of `+` in container tags; version strings longer than 128 characters cannot be published to GHCR. After the images are verified, the publisher uploads the single `compose.yaml` and its checksum list, rendered for that release. Compose pins the ingress image by its registry digest; initialization rejects an image whose build revision differs from the Compose revision. A draft Release stays unpublished. The combined build/publication job uses `GITHUB_TOKEN` with `packages: write`. On the first publication, GitHub creates each container package as private: a package administrator must change all four packages to **Public** in their package settings before users can pull anonymously. See [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). Verify an unauthenticated pull after changing visibility. Repository visibility alone does not make a new container package public. diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index f713c20b..2ef6abe1 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: 89ce54ec713de69bbc2d8aaa89ddb577744d0ca694c3fdbca8bf2a452200ad46 +source_hash: e697e320a2df0c130cb004f5e8bbda50deff6515ef7727bea066fc0d4f688b7d --- Core 安装的每项设置都恰好只有一个归属位置。共有两类： @@ -48,9 +48,8 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 | Variable | Default | Meaning | | --- | --- | --- | | `OAC_PUBLIC_URL` | `http://localhost:8080` | 应用、节点、沙箱和自托管执行器使用的源地址。参阅[修改公开 URL](#changing-the-public-url) | -| `OAC_HOST` | `127.0.0.1` | Address published by `ports.yaml`. `install.sh` sets `0.0.0.0` | +| `OAC_HOST` | `127.0.0.1` | `compose.yaml` 发布的 Web 绑定地址。`install.sh` 设置为 `0.0.0.0` | | `OAC_WEB_PORT` | `8080` | Host port of Web | -| `COMPOSE_FILE` | `compose.yaml:ports.yaml` | Compose 文件。`ports.yaml` 发布 Web；托管平台省略它 | | `OAC_LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error` | | `OAC_LOG_FORMAT` | `auto` | `auto`, `text` or `json` | | `OAC_LOG_ADD_SOURCE` | unset | `1` adds source locations | @@ -128,7 +127,7 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 | 路径 | 内容 | 修改者 | | --- | --- | --- | | `.env` | [进程设置](#process-settings-configjson)。由你编辑的文件 | 你，然后运行 `oac apply`；托管域名设置写入 `OAC_PUBLIC_URL` | -| `compose.yaml`、`ports.yaml` | 发行版的服务定义。不要编辑 | 发行版 | +| `compose.yaml` | 发行版的服务定义。不要编辑 | 发行版 | | `oac` | [管理命令](getting-started/operations.md#the-oac-command)，从 Core 镜像复制 | 安装程序 | | `data/secrets/web/core.key` | [Core 密钥](getting-started/operations.md#core-key) | `oac rotate-core-key` | | `data/secrets/core/credential.key` | 加密 Core 在数据库中封存内容的密钥 | 无。必须与数据库一同保留 | diff --git a/docs/zh/getting-started/install-options.md b/docs/zh/getting-started/install-options.md index 52e0a141..b0faa317 100644 --- a/docs/zh/getting-started/install-options.md +++ b/docs/zh/getting-started/install-options.md @@ -1,7 +1,7 @@ --- title: "安装选项与高级部署" source: docs/getting-started/install-options.md -source_hash: 93e842b2818adb1ca56bdca911fd0c828868ce39c70378e4430b296865cd962b +source_hash: e063d7dcd615ef5d6ba8b43f1cbe5a11c75325925605f376161f50b9383297c0 --- [默认安装](install.md)无需任何选项。使用本页可以在现有反向代理后运行，或者在无法访问互联网时进行安装。 @@ -20,11 +20,11 @@ source_hash: 93e842b2818adb1ca56bdca911fd0c828868ce39c70378e4430b296865cd962b 在 Linux amd64 上使用发行版中的 `compose.yaml` 和 Docker Compose 2.26 或更高版本。发行流程会在 [Compose 模板](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml)中固定初始化镜像及源码版本。Core 和 Web 使用 `latest` 镜像，PostgreSQL 使用 `postgres:16-alpine`。它会启动 PostgreSQL、Core 和 Web。Web 把 `/v1` 和 `/api/v1` 转发到 Core。数据通过目录 bind mount 挂载。一次性初始化服务会在该目录中生成随机机密信息并准备节点安装程序；Core 启动时执行数据库迁移。[Compose 配置](../configuration.md#compose-installations)负责管理各项设置和数据目录。 -进行本地试用时，请将同一发行版的 `compose.yaml` 和 `ports.yaml` 下载到同一个目录，然后运行： +进行本地试用时，请将发行版的 `compose.yaml` 下载到一个空目录，然后运行： ```sh -docker compose -f compose.yaml -f ports.yaml up -d --wait --wait-timeout 900 -docker compose -f compose.yaml exec web oac-web core-key +docker compose up -d --wait --wait-timeout 900 +docker compose exec web oac-web core-key ``` `oac-web core-key` 会将生成的 Core 密钥打印到终端，而不会将其写入容器日志。打开 `http://localhost:8080` 并使用该密钥登录。所有安装机密信息都会自动生成；重启时请保留同一个 Compose 项目及其数据目录。 @@ -35,11 +35,11 @@ docker compose -f compose.yaml exec web oac-web core-key ### Dokploy {#dokploy} -创建一个 Docker Compose 应用并粘贴 `compose.yaml`。将 `OAC_PUBLIC_URL` 设置为公共 HTTPS 源地址，启用隔离部署，并为 `web` 服务添加域名和端口 `8080`。部署前，为此域名启用 **HTTPS**，并选择 **Let's Encrypt** 等证书提供程序。部署时不要使用 `ports.yaml`；内部服务不会发布主机端口。将此 Compose 文件打包到 Dokploy 模板目录时，[模板元数据](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/dokploy.toml)会提供生成的域名和环境；导入后仍需启用 HTTPS 及其证书提供程序。 +创建一个 Docker Compose 应用并粘贴 `compose.yaml`。将 `OAC_PUBLIC_URL` 设置为公共 HTTPS 源地址，启用隔离部署，并为 `web` 服务添加域名和端口 `8080`。部署前，为此域名启用 **HTTPS**，并选择 **Let's Encrypt** 等证书提供程序。部署前移除 `web` 服务的 `ports` 块，让 Dokploy 路由到容器端口，而不发布宿主机端口。将此 Compose 文件打包到 Dokploy 模板目录时，[模板元数据](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/dokploy.toml)会提供生成的域名和环境；导入后仍需启用 HTTPS 及其证书提供程序。 ### Coolify {#coolify} -创建一个 **Docker Compose Empty** 服务并粘贴 `compose.yaml`。将 `OAC_PUBLIC_URL` 设置为公共 HTTPS 源地址，并将该域名分配给 `web` 服务的端口 `8080`。将 Coolify 的 `exclude_from_hc: true` 添加到 `init` 的服务定义中，使已完成的初始化不会影响其总体健康状态。保存并在不包含 `ports.yaml` 的情况下部署；HTTPS 由 Coolify 提供。 +创建一个 **Docker Compose Empty** 服务并粘贴 `compose.yaml`。将 `OAC_PUBLIC_URL` 设置为公共 HTTPS 源地址，并将该域名分配给 `web` 服务的端口 `8080`。将 Coolify 的 `exclude_from_hc: true` 添加到 `init` 的服务定义中，使已完成的初始化不会影响其总体健康状态。移除 `web` 服务的 `ports` 块，再保存并部署；Coolify 路由到容器端口并提供 HTTPS。 在这两个平台上，打开服务器终端并运行 `docker compose ls`，查找已部署的项目名称和 Compose 文件。使用这些完全一致的值以及该部署的 `OAC_PUBLIC_URL`，运行 `docker compose -p -f exec web oac-web core-key`，然后在已配置的源地址登录。[Dokploy 域名指南](https://docs.dokploy.com/docs/core/docker-compose/domains)和[Coolify Compose 指南](https://coolify.io/docs/services/configuration/docker-compose)介绍了各自的域和服务控制项。这些都是可导入的部署文件；本仓库不发布托管市场条目。 diff --git a/docs/zh/maintainers.md b/docs/zh/maintainers.md index efe6873f..e2af80ad 100644 --- a/docs/zh/maintainers.md +++ b/docs/zh/maintainers.md @@ -1,7 +1,7 @@ --- title: "构建并发布 OpenAgentCore" source: docs/maintainers.md -source_hash: e49645e98eac4e584dabd59b2b90c4e89b0c0fd521b5e1be9829ad3286447bd6 +source_hash: c7a280c8367f0b86b4ccc4eeb3e3ee203519804ef5e0a09f874a71b0e78ac6be --- 本指南面向负责构建和发布 OpenAgentCore 的维护者。要安装 Core 和 Web，请使用 [安装指南](getting-started/install.md)。安装器代码遵循的规则见 [部署](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/README.md) 和 [节点安装器](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/node/README.md)；必需检查见 [CONTRIBUTING](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/CONTRIBUTING.md#required-checks)。 @@ -134,7 +134,7 @@ git push origin v1.2.3 ### 容器注册表 {#container-registry} -版本发布和手动的 `build-` 草稿都会将 Linux amd64 镜像发布为 `ghcr.io/minimax-ai/openagentcore/:`，其中 `` 为 `core`、`web`、`runtime` 或 `ingress`。例如，`ghcr.io/minimax-ai/openagentcore/core:v1.2.3`。草稿使用标签 `build-`。PostgreSQL 使用其上游镜像，不会重新发布。注册表镜像从发布归档中加载，不会重新构建。仅当现有版本标签的镜像配置摘要与本次发布相同时才复用该标签；如果镜像不同，则停止发布。稳定版还会把每个组件的 `latest` 标签移到该镜像。预发布和草稿不会改动 `latest`。SemVer 构建元数据在容器标签中使用 `_` 代替 `+`；长度超过 128 个字符的版本字符串无法发布到 GHCR。镜像验证之后，发布器会上传为该发行版渲染的 `compose.yaml` 和 `ports.yaml` 及其校验和。Compose 使用注册表摘要固定 ingress 镜像；如果镜像构建版本与 Compose 版本不同，初始化会拒绝运行。草稿 Release 保持未发布。 +版本发布和手动的 `build-` 草稿都会将 Linux amd64 镜像发布为 `ghcr.io/minimax-ai/openagentcore/:`，其中 `` 为 `core`、`web`、`runtime` 或 `ingress`。例如，`ghcr.io/minimax-ai/openagentcore/core:v1.2.3`。草稿使用标签 `build-`。PostgreSQL 使用其上游镜像，不会重新发布。注册表镜像从发布归档中加载，不会重新构建。仅当现有版本标签的镜像配置摘要与本次发布相同时才复用该标签；如果镜像不同，则停止发布。稳定版还会把每个组件的 `latest` 标签移到该镜像。预发布和草稿不会改动 `latest`。SemVer 构建元数据在容器标签中使用 `_` 代替 `+`；长度超过 128 个字符的版本字符串无法发布到 GHCR。镜像验证之后，发布器会上传为该发行版渲染的单个 `compose.yaml` 及其校验和清单。Compose 使用注册表摘要固定 ingress 镜像；如果镜像构建版本与 Compose 版本不同，初始化会拒绝运行。草稿 Release 保持未发布。 合并的构建/发布作业使用具有 `packages: write` 权限的 `GITHUB_TOKEN`。首次发布时，GitHub 会将每个容器软件包创建为私有：软件包管理员必须先在各自的软件包设置中将全部四个软件包改为 **Public**，用户才能匿名拉取。请参阅 [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry)。更改可见性后，请验证未认证拉取。仅更改仓库可见性并不会使新的容器软件包变为公开。 diff --git a/scripts/ci_plan_test.py b/scripts/ci_plan_test.py index 8784e371..361bf7e5 100644 --- a/scripts/ci_plan_test.py +++ b/scripts/ci_plan_test.py @@ -23,7 +23,7 @@ def test_installer_does_not_download_a_browser_or_run_database_tests(self): self.assertEqual(self.jobs("deploy/install.sh", "deploy/node/node_payload.py"), {"hygiene", "distribution"}) def test_compose_inputs_select_live_and_fixture_checks_without_image_builds(self): - for path in ("deploy/compose/compose.yaml", "deploy/compose/ports.yaml", "deploy/compose/https.yaml", "deploy/compose/dokploy.toml", + for path in ("deploy/compose/compose.yaml", "deploy/compose/https.yaml", "deploy/compose/dokploy.toml", "scripts/compose-smoke.py", "scripts/render-compose.py", "deploy/compose/test_compose.py"): self.assertEqual(self.jobs(path), {"hygiene", "distribution", "compose"}) self.assertFalse(ci.select([path])["image"]) diff --git a/scripts/compose-smoke.py b/scripts/compose-smoke.py index 4d599250..2a6b27fa 100644 --- a/scripts/compose-smoke.py +++ b/scripts/compose-smoke.py @@ -109,16 +109,12 @@ def main(): 'REVISION': pins['revision'], 'INIT_IMAGE': 'ghcr.io/minimax-ai/openagentcore/ingress@sha256:' + '0' * 64, })) - override = directory / 'ports.json' + override = directory / 'offline-init.json' images = build_images(directory, project.removeprefix('oac-smoke-')) - def publish(port): - override.write_text(json.dumps({'services': {'init': {'network_mode': 'none'}, 'web': {'ports': [ - {'target': 8080, 'published': str(port), 'host_ip': '127.0.0.1'}, - ]}}})) - - publish(0) + override.write_text(json.dumps({'services': {'init': {'network_mode': 'none'}}})) env = {**os.environ, 'COMPOSE_PROGRESS': 'plain', 'OAC_DATA_DIR': str(data), + 'OAC_HOST': '127.0.0.1', 'OAC_WEB_PORT': '0', **{'OAC_IMAGE_' + name.upper(): image for name, image in images.items()}} env.pop('OAC_PUBLIC_URL', None) command = ['docker', 'compose', '--env-file', os.devnull, '-p', project, @@ -201,7 +197,7 @@ def terminate(_signum, _frame): print('Configuring a reachable URL and recreating containers with the same data directory', flush=True) # Retain the assigned port across recreation, without claiming a fixed host port. - publish(address.rsplit(':', 1)[1]) + env['OAC_WEB_PORT'] = address.rsplit(':', 1)[1] env['OAC_PUBLIC_URL'] = address compose('down') compose('up', '-d', '--wait', '--wait-timeout', '120', timeout=180) @@ -224,6 +220,9 @@ def terminate(_signum, _frame): raise finally: compose('down', '--volumes', '--remove-orphans', timeout=60) + # Match the host installer's cleanup without requiring tools in scratch init. + compose('run', '--rm', '--no-deps', '--volume', str(data) + ':/data', + '--entrypoint', 'find', 'database', '/data', '-mindepth', '1', '-delete') subprocess.run(['docker', 'image', 'rm', '-f', *images.values()], capture_output=True, timeout=60) diff --git a/scripts/publish-core-release.test.py b/scripts/publish-core-release.test.py index dd5d736e..29ff5258 100644 --- a/scripts/publish-core-release.test.py +++ b/scripts/publish-core-release.test.py @@ -68,7 +68,7 @@ def test_draft_publishes_images_and_stays_unpublished(self): self.publish(tag="build-" + self.revision, mode="draft") self.images.assert_called_once() self.assertTrue(self.release["draft"]) - self.assertEqual(len(self.release["assets"]), 15) + self.assertEqual(len(self.release["assets"]), 14) def test_missing_native_asset_refuses_release_creation(self): (self.assets / f"oac-native-{self.revision}-windows-amd64.tar.gz").unlink() @@ -142,7 +142,7 @@ def response(repo, endpoint, *args): return result if endpoint == "releases/7": self.assertEqual(active, 0) - self.assertIn(len(self.release["assets"]), (12, 15)) + self.assertIn(len(self.release["assets"]), (12, 14)) return self.response(repo, endpoint, *args) self.api.side_effect = response self.publish() @@ -153,7 +153,8 @@ def test_version_tag_publishes_complete_fixed_id(self): self.publish() self.assertFalse(self.release["draft"]) self.assertFalse(self.release["prerelease"]) - self.assertEqual(len(self.release["assets"]), 15) + self.assertEqual(len(self.release["assets"]), 14) + self.assertEqual({a["name"] for a in self.release["assets"] if a["name"].endswith(".yaml")}, {"compose.yaml"}) self.assertEqual(self.api.call_args.args[1:], ("releases/7", "--method", "PATCH", "-F", "draft=false")) diff --git a/scripts/render-compose.py b/scripts/render-compose.py index 6e2b4871..cc88c4f2 100644 --- a/scripts/render-compose.py +++ b/scripts/render-compose.py @@ -10,7 +10,6 @@ ROOT = pathlib.Path(__file__).resolve().parents[1] TEMPLATE = ROOT / "deploy/compose/compose.yaml" -PORTS = ROOT / "deploy/compose/ports.yaml" TOKENS = ("REVISION", "INIT_IMAGE") @@ -39,11 +38,10 @@ def render(values): def write_assets(directory, values): - """Write compose.yaml, the port files and one checksum list for them.""" + """Write compose.yaml and its checksum list.""" directory = pathlib.Path(directory) files = { "compose.yaml": render(values).encode(), - "ports.yaml": PORTS.read_bytes(), } written, lines = [], [] for name, data in files.items():