From c2b511da564138f2715f2c3da0d041577e9ab7ee Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 1 Oct 2026 14:37:08 +0900 Subject: [PATCH 1/2] build(deps): link self_encryption only with test-utils The default `webrtc-direct` feature enabled `dep:self_encryption`, but the only code that calls it is `Devnet::publish_public_file`, which is compiled only with `test-utils` (it seeds public files for the browser devnet tests). Release builds therefore compiled self_encryption without using it, and the crate appeared in the dependency graph of every shipped binary. The published self_encryption releases are GPL-3.0 with a linking exception, so licence scanners report it against ant-node even though no code reaches the binary. Move the optional dependency to `test-utils`. Default and release builds no longer resolve self_encryption on any release target; `test-utils` builds and the five-node browser devnet test that self-encrypts a file are unchanged. No public API changes: the one function that uses the crate already required `test-utils`. --- Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a129dd58..5554f187 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -112,6 +112,8 @@ bao = "0.13.1" # Shared portable browser profile. The native listener is enabled separately # by the `webrtc-direct` feature. saorsa-transport = { version = "0.37.0", default-features = false, features = ["webrtc"] } +# Only the test-utils devnet helper that seeds public files for browser tests +# uses it, so release builds do not link it. self_encryption = { version = "0.36", optional = true } [target.'cfg(unix)'.dependencies] @@ -219,13 +221,12 @@ default = ["logging", "webrtc-direct"] logging = ["tracing", "tracing-subscriber", "tracing-appender"] # Expose test helpers (cache_insert, payment_verifier accessor) for # integration tests and downstream test harnesses. -test-utils = [] +test-utils = ["dep:self_encryption"] # Direct browser transport from ADR-0015. Enabled by default; minimal # native-only builds can omit it with `--no-default-features`. webrtc-direct = [ "saorsa-transport/default", "saorsa-transport/webrtc-direct", - "dep:self_encryption", ] [profile.release] From 638a5b54e93787b31d6ca8b8db07edca4c55c6c3 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 1 Oct 2026 14:39:22 +0900 Subject: [PATCH 2/2] ci(release): ship licence texts and third-party notices in archives Release archives contained only the binary and bootstrap_peers.toml. The binary statically links several hundred third-party crates and some bundled C libraries whose licences (MIT, BSD, ISC, Apache-2.0, MPL-2.0, OpenLDAP and others) require their copyright and licence notices to accompany binary redistribution, and the archives did not carry ant-node's own licence files. scripts/third_party_notices/generate.py builds THIRD-PARTY-NOTICES.txt from Cargo.lock for one or more targets. It takes the packages `cargo tree` resolves (normal and build dependencies, so build tools are listed too), matches each to exactly one `cargo metadata` package, and for each: - reproduces, unaltered, every licence, copyright, notice, authors, patents, credits and third-party file the crate ships at any depth, printing identical texts once, and names the crate's exact crates.io source archive, which is also how source is offered where a licence requires it (MPL-2.0); - checks the declared licence against the policy in config.toml and records the licence it is redistributed under, which must be allowed and whose text must be identified in one of the crate's own licence files (those of bundled components do not count); - when a crate ships no such files, or none with that licence's text, takes them from its repository at the commit recorded in the crate (or Cargo's checkout, for a git dependency), and failing that from a reviewed, version-pinned config entry with a checksummed canonical text (saorsa-pqc, siphasher, lmdb-master-sys and five macOS objc2 crates today); - requires a reviewed config entry for any crate that may contain native code: one that ships C, C++, Objective-C, CUDA or assembly sources or prebuilt libraries, builds with cc/cmake/nasm, declares `links`, or is a `*-src` crate (LMDB under the OpenLDAP Public License 2.8, mimalloc, zstd, bzip2, XZ Utils, AWS-LC, ring, BLAKE3, and a few build tools and test fixtures). Any gap fails generation. For the statically linked Linux builds it also appends musl's COPYRIGHT for the musl release the building rustc uses: the version is read from Rust's own build scripts at that rustc's commit, and the text from that musl release tarball. Each release build job generates the notices for its own target and puts them in the archive with LICENSE-MIT, LICENSE-APACHE and RUST-STD-COPYRIGHT.html, the Rust standard library's notices copied from the toolchain that built the binary. The signed Windows archive is repackaged with the same files. The upgrade path and the client's node installer extract the binary by name, so the extra entries do not affect them. CI generates the notices for all release targets on every pull request. --- .github/workflows/ci.yml | 25 + .github/workflows/release.yml | 45 +- .gitignore | 3 + scripts/third_party_notices/config.toml | 162 ++++++ scripts/third_party_notices/generate.py | 696 ++++++++++++++++++++++++ 5 files changed, 927 insertions(+), 4 deletions(-) create mode 100644 scripts/third_party_notices/config.toml create mode 100644 scripts/third_party_notices/generate.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bf4bc96..9c2120d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,31 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: cargo clippy --all-targets --all-features -- -D warnings + # The release workflow ships THIRD-PARTY-NOTICES.txt in every archive. + # Generating it on each pull request catches a dependency that ships no + # licence text, or a new crate that compiles bundled C code, before release. + notices: + name: Third-party notices + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Generate THIRD-PARTY-NOTICES.txt + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + python scripts/third_party_notices/generate.py + --config scripts/third_party_notices/config.toml + --output THIRD-PARTY-NOTICES.txt + --target x86_64-unknown-linux-musl + --target aarch64-unknown-linux-musl + --target x86_64-apple-darwin + --target aarch64-apple-darwin + --target x86_64-pc-windows-msvc + test: name: Test (${{ matrix.os }}) runs-on: ${{ matrix.os }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b4cf880d..4746e699 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -125,6 +125,25 @@ jobs: with: key: ${{ matrix.target }} + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # Collects, unaltered, the licence, copyright and notice files of every + # crate this target's build resolves, including build-time tools, and + # fails if any crate's licence is outside the policy or its texts cannot + # be found (see the script and its config). The token raises the GitHub + # rate limit for licence files fetched from crates' repositories. + - name: Generate THIRD-PARTY-NOTICES.txt + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + python scripts/third_party_notices/generate.py + --config scripts/third_party_notices/config.toml + --output THIRD-PARTY-NOTICES.txt + --target ${{ matrix.target }} + - name: Install cross (Linux ARM64) if: matrix.cross run: cargo install cross --git https://github.com/cross-rs/cross @@ -140,21 +159,30 @@ jobs: if: ${{ !matrix.cross }} run: cargo build --release --target ${{ matrix.target }} + # Archives carry the project's licence files, the third-party notices and + # the Rust standard library's notices from the toolchain that built the + # binary. The upgrade path extracts only the binary by name, so extra + # files do not affect it. - name: Create archive (Unix) if: matrix.archive == 'tar.gz' run: | - cp config/bootstrap_peers.toml target/${{ matrix.target }}/release/ + cp config/bootstrap_peers.toml LICENSE-MIT LICENSE-APACHE THIRD-PARTY-NOTICES.txt target/${{ matrix.target }}/release/ + cp "$(rustc --print sysroot)/share/doc/rust/COPYRIGHT-library.html" target/${{ matrix.target }}/release/RUST-STD-COPYRIGHT.html cd target/${{ matrix.target }}/release - tar -czvf ../../../ant-node-cli-${{ matrix.friendly_name }}.tar.gz ${{ matrix.binary }} bootstrap_peers.toml + tar -czvf ../../../ant-node-cli-${{ matrix.friendly_name }}.tar.gz ${{ matrix.binary }} bootstrap_peers.toml LICENSE-MIT LICENSE-APACHE THIRD-PARTY-NOTICES.txt RUST-STD-COPYRIGHT.html cd ../../.. - name: Create archive (Windows) if: matrix.archive == 'zip' shell: pwsh run: | - Copy-Item "config/bootstrap_peers.toml" "target/${{ matrix.target }}/release/bootstrap_peers.toml" + foreach ($file in "config/bootstrap_peers.toml", "LICENSE-MIT", "LICENSE-APACHE", "THIRD-PARTY-NOTICES.txt") { + Copy-Item $file "target/${{ matrix.target }}/release/" -ErrorAction Stop + } + $sysroot = (rustc --print sysroot).Trim() + Copy-Item (Join-Path $sysroot "share/doc/rust/COPYRIGHT-library.html") "target/${{ matrix.target }}/release/RUST-STD-COPYRIGHT.html" -ErrorAction Stop Push-Location "target/${{ matrix.target }}/release" - Compress-Archive -Path "${{ matrix.binary }}", "bootstrap_peers.toml" -DestinationPath "../../../ant-node-cli-${{ matrix.friendly_name }}.zip" + Compress-Archive -Path "${{ matrix.binary }}", "bootstrap_peers.toml", "LICENSE-MIT", "LICENSE-APACHE", "THIRD-PARTY-NOTICES.txt", "RUST-STD-COPYRIGHT.html" -DestinationPath "../../../ant-node-cli-${{ matrix.friendly_name }}.zip" Pop-Location - name: Upload artifact @@ -267,6 +295,13 @@ jobs: mkdir "$staging" cp artifacts/ant-node.exe "$staging/" cp config/bootstrap_peers.toml "$staging/" + for file in LICENSE-MIT LICENSE-APACHE THIRD-PARTY-NOTICES.txt RUST-STD-COPYRIGHT.html; do + if [ ! -f "artifacts/$file" ]; then + echo "::error::$file not found in the unsigned archive" + exit 1 + fi + cp "artifacts/$file" "$staging/" + done (cd "$staging" && 7z a "../${staging}.zip" ./*) @@ -405,6 +440,8 @@ jobs: | macOS ARM64 (Apple Silicon) | `ant-node-cli-macos-arm64.tar.gz` | | Windows x64 | `ant-node-cli-windows-x64.zip` | + Each archive also contains `LICENSE-MIT` and `LICENSE-APACHE`, plus `THIRD-PARTY-NOTICES.txt` and `RUST-STD-COPYRIGHT.html` with the licences and copyright notices of the third-party code in the binary and of the tools used to build it. + **CLI Usage:** ```bash # Linux/macOS — extract and run (bootstrap peers auto-discovered) diff --git a/.gitignore b/.gitignore index e739a592..2219ab68 100644 --- a/.gitignore +++ b/.gitignore @@ -230,3 +230,6 @@ proptest-regressions/ !.claude/commands/ .cache/ /devnet-manifest.json + +# Generated by scripts/third_party_notices/generate.py; the release workflow builds it. +/THIRD-PARTY-NOTICES.txt diff --git a/scripts/third_party_notices/config.toml b/scripts/third_party_notices/config.toml new file mode 100644 index 00000000..b4d12ce6 --- /dev/null +++ b/scripts/third_party_notices/config.toml @@ -0,0 +1,162 @@ +# Settings for generate.py, which the release workflow runs to produce the +# THIRD-PARTY-NOTICES.txt shipped in every ant-node archive. + +header = """ +ant-node is licensed under MIT OR Apache-2.0; see LICENSE-MIT and +LICENSE-APACHE. This file covers the third-party software in this build and +the tools used to produce it. Each crate's Source link is its exact source, +which is how source is made available where a licence requires it. +""" + +# Licences the release may redistribute third-party code under, most preferred +# first. A crate offering a choice is redistributed under the first allowed +# option; a crate whose licence allows no option here fails generation. +allowed = [ + "MIT", + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "Zlib", + "0BSD", + "MIT-0", + "CC0-1.0", + "Unlicense", + "BSL-1.0", + "Unicode-3.0", + "Unicode-DFS-2016", + "CDLA-Permissive-2.0", + "MPL-2.0", +] + +# Crates that may contain native code: they ship C, C++, Objective-C, CUDA or +# assembly sources or prebuilt libraries, build with cc/cmake/nasm, declare +# `links`, or are a `*-src` crate. Licence files anywhere in a crate are +# reproduced automatically; each entry records what was found on review, and +# can add licence files with other names through `files` (relative to the +# crate root, globs allowed), which count as a bundled component's unless +# `files_govern = true` marks them as the crate's own. An entry with a +# `version` applies to that release only. A crate that needs an entry and has +# none fails generation. + +[native.lmdb-master-sys] +note = "bundles LMDB (Howard Chu, Symas Corp.) under the OpenLDAP Public License 2.8; its COPYRIGHT and LICENSE are reproduced." + +[native.libmimalloc-sys] +note = "bundles mimalloc (Microsoft Corporation, Daan Leijen) under MIT; its LICENSE files are reproduced." + +[native.zstd-sys] +note = "bundles zstd (Meta Platforms) under BSD-3-Clause (zstd/LICENSE), chosen from its BSD-3-Clause OR GPL-2.0 licence; zstd/COPYING is the GPL-2.0 alternative, reproduced only because the crate ships it." + +[native.bzip2-sys] +note = "bundles bzip2 (Julian Seward) under its BSD-style licence; its LICENSE is reproduced." + +[native.lzma-sys] +note = "bundles liblzma from XZ Utils, which is in the public domain (xz-*/COPYING); the GPL and LGPL texts next to it cover XZ Utils tools and scripts that lzma-sys does not build, and are reproduced only because the crate ships them." + +[native.aws-lc-sys] +note = "bundles AWS-LC; its LICENSE lists each component it derives from (BoringSSL, OpenSSL, and others) with their copyright notices." + +[native.aws-lc-rs] +note = "declares a `links` key but builds no native code of its own; AWS-LC comes from aws-lc-sys." + +[native.ring] +note = "its C and assembly code derives from BoringSSL; LICENSE, LICENSE-BoringSSL and LICENSE-other-bits cover it." + +[native.blake3] +note = "the C and assembly implementations are part of BLAKE3 and share the crate's licence." + +[native.cc] +note = "a build tool; its one C file is compiled on the build machine to detect the C compiler, and nothing from it is linked." + +[native.syn-solidity] +note = "a proc-macro dependency; src/ident/kw.c is a developer script that generated its keyword tables, and is not compiled." + +[native.nix] +note = "its only C file is a kernel-module test fixture that is not compiled into dependants." + +[native.windows_x86_64_msvc] +note = "ships the Windows import library that windows-rs links against on Windows builds; it is covered by the crate's MIT OR Apache-2.0 licence (Microsoft Corporation)." + +[native.saorsa-transport] +version = "0.37.0" +note = "the published crate also contains a .minimax/ folder of AI-assistant files (Python tools and .NET DLLs) committed to its repository by mistake; nothing in it is compiled or linked into this build." + +# Explanations printed with a crate's entry, for licence or notice files that +# could mislead without context. + +[remark.security-framework] +note = "its THIRD_PARTY file covers documentation adapted from Apple's Security Framework under the Apple Public Source License 2.0; that documentation is not compiled into this build." + +# Crates whose published files and repository contain no text of any licence +# they are offered under. Each entry is for one reviewed version; it supplies +# the canonical text of one of those licences, pinned by checksum, and says +# why it is needed. Remove an entry once the crate ships the text; generation +# prints a note when one goes unused. + +[missing.saorsa-pqc] +version = "0.5.2" +text_url = "https://www.apache.org/licenses/LICENSE-2.0.txt" +text_sha256 = "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30" +note = "declares MIT OR Apache-2.0 but its published crate and repository contain no licence text; it is redistributed under Apache-2.0, whose text is reproduced from apache.org." + +[missing.siphasher] +version = "1.0.3" +text_url = "https://raw.githubusercontent.com/spdx/license-list-data/v3.29.0/text/MIT.txt" +text_sha256 = "b05785f9f18e6716bab63424b11454513b9943a222595b70411009202fc592b5" +note = "its COPYING, reproduced with its copyright lines, refers to LICENSE-MIT and LICENSE-APACHE files that neither the crate nor its repository contain; the MIT text is reproduced from the SPDX licence list." + +[missing.lmdb-master-sys] +version = "0.2.6" +text_url = "https://www.apache.org/licenses/LICENSE-2.0.txt" +text_sha256 = "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30" +note = "declares Apache-2.0 for its Rust bindings but ships only LMDB's licence; the Apache-2.0 text is reproduced from apache.org." + +[missing.objc2] +version = "0.6.4" +text_url = "https://raw.githubusercontent.com/spdx/license-list-data/v3.29.0/text/MIT.txt" +text_sha256 = "b05785f9f18e6716bab63424b11454513b9943a222595b70411009202fc592b5" +note = "its repository's LICENSE.md, reproduced, names MIT without its text or a copyright line; the MIT text is reproduced from the SPDX licence list." + +[missing.block2] +version = "0.6.2" +text_url = "https://raw.githubusercontent.com/spdx/license-list-data/v3.29.0/text/MIT.txt" +text_sha256 = "b05785f9f18e6716bab63424b11454513b9943a222595b70411009202fc592b5" +note = "its repository's LICENSE.md, reproduced, names MIT without its text or a copyright line; the MIT text is reproduced from the SPDX licence list." + +[missing.objc2-encode] +version = "4.1.0" +text_url = "https://raw.githubusercontent.com/spdx/license-list-data/v3.29.0/text/MIT.txt" +text_sha256 = "b05785f9f18e6716bab63424b11454513b9943a222595b70411009202fc592b5" +note = "its repository's LICENSE.md, reproduced, names MIT without its text or a copyright line; the MIT text is reproduced from the SPDX licence list." + +[missing.objc2-foundation] +version = "0.3.2" +text_url = "https://raw.githubusercontent.com/spdx/license-list-data/v3.29.0/text/MIT.txt" +text_sha256 = "b05785f9f18e6716bab63424b11454513b9943a222595b70411009202fc592b5" +note = "its repository's LICENSE.md, reproduced, names MIT without its text or a copyright line; the MIT text is reproduced from the SPDX licence list." + +[missing.objc2-core-foundation] +version = "0.3.2" +text_url = "https://raw.githubusercontent.com/spdx/license-list-data/v3.29.0/text/MIT.txt" +text_sha256 = "b05785f9f18e6716bab63424b11454513b9943a222595b70411009202fc592b5" +note = "its repository's LICENSE.md, reproduced, names MIT without its text or a copyright line; the MIT text is reproduced from the SPDX licence list." + +[[appendix]] +title = "Rust standard library" +intro = """ +Every Rust program links the Rust standard library. Its copyright and licence +notices, copied from the toolchain that built this binary, are in +RUST-STD-COPYRIGHT.html next to this file. +""" + +[[appendix]] +title = "musl libc" +targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"] +intro = """ +This Linux build is statically linked against musl libc as shipped with Rust's +musl targets. The musl release the building toolchain uses, and that release's +COPYRIGHT file, follow. +""" +musl_from_rustc = true diff --git a/scripts/third_party_notices/generate.py b/scripts/third_party_notices/generate.py new file mode 100644 index 00000000..b3bf57ed --- /dev/null +++ b/scripts/third_party_notices/generate.py @@ -0,0 +1,696 @@ +#!/usr/bin/env python3 +"""Generate THIRD-PARTY-NOTICES.txt for a Rust release build. + +The notices cover every package `cargo tree` resolves for the build (normal and +build dependencies, for the given targets and features), so they over-cover +rather than miss: build tools are listed too. For each package: + +- its declared licence must be allowed by the config's policy, and the text of + the licence it is redistributed under must be in one of its crate-level + licence files (licence files of bundled components do not count); + for a choice of licences (`OR`) the notices record which one applies; +- every licence, copyright, notice and authors file it ships, at any depth, is + reproduced unaltered, identical texts once; +- when a package ships no such files, or none containing the licence it is + redistributed under, its repository is searched at the exact commit + recorded in its `.cargo_vcs_info.json` (or, for a git dependency, Cargo's + checkout at the locked commit), and failing that a reviewed config entry + may supply the canonical text for that one version; +- a package that may contain native code (it ships C, C++ or assembly sources + or prebuilt libraries, has a cc/cmake/nasm build dependency, a `links` key, + or is a `*-src` crate) needs a reviewed config entry saying what it bundles. + +Any gap fails generation. Requires Python 3.11+, cargo and a Cargo.lock. Set +GITHUB_TOKEN to raise the GitHub API rate limit. +""" + +import argparse +import glob +import hashlib +import http.client +import io +import json +import os +import re +import subprocess +import sys +import tarfile +import time +import tomllib +import urllib.error +import urllib.request + +LICENCE_FILE = re.compile( + r"^(LICEN[CS]ES?|COPYING|COPYRIGHT|NOTICES?|UNLICENSE|AUTHORS|PATENTS?|CREDITS" + r"|ATTRIBUTIONS?|THIRD[-_ ]?PARTY[-_ A-Z]*)([-._ ].*)?$", + re.I, +) +NATIVE_BUILD_DEPS = {"cc", "cmake", "nasm-rs", "autotools"} +NATIVE_FILE = re.compile( + r"\.(c|cc|cpp|cxx|c\+\+|m|mm|cu|s|asm|a|lib|o|obj|so|dylib|dll)$", re.I +) +# Phrases that identify a licence text, compared case-insensitively with +# whitespace collapsed: a text must contain every `has` phrase and none of the +# `lacks` phrases. A licence a crate is redistributed under must be identified +# in one of its reproduced files; for `X WITH Y` both parts must be. +LICENCE_MARKERS = { + "MIT": {"has": ["permission is hereby granted, free of charge", + "the above copyright notice and this permission notice shall be included"]}, + "MIT-0": {"has": ["permission is hereby granted, free of charge"], + "lacks": ["the above copyright notice and this permission notice shall be included"]}, + "Apache-2.0": {"has": ["apache license", "version 2.0", + "terms and conditions for use, reproduction, and distribution"]}, + "BSD-2-Clause": {"has": ["redistribution and use in source and binary forms", + "redistributions in binary form must reproduce"], + "lacks": ["neither the name"]}, + "BSD-3-Clause": {"has": ["redistribution and use in source and binary forms", + "redistributions in binary form must reproduce", "neither the name"]}, + "ISC": {"has": ["permission to use, copy, modify, and", + "provided that the above copyright notice and this permission notice appear"]}, + "0BSD": {"has": ["permission to use, copy, modify, and/or distribute this software for any purpose"], + "lacks": ["provided that the above copyright notice"]}, + "Zlib": {"has": ["without any express or implied warranty", + "altered source versions must be plainly marked"]}, + "CC0-1.0": {"has": ["cc0 1.0 universal"]}, + "Unlicense": {"has": ["free and unencumbered software released into the public domain"]}, + "BSL-1.0": {"has": ["boost software license - version 1.0"]}, + "Unicode-3.0": {"has": ["unicode license v3"]}, + "Unicode-DFS-2016": {"has": ["unicode, inc. license agreement - data files and software"]}, + "CDLA-Permissive-2.0": {"has": ["community data license agreement - permissive - version 2.0"]}, + "MPL-2.0": {"has": ["mozilla public license version 2.0"]}, + "GPL-3.0": {"has": ["gnu general public license", "version 3, 29 june 2007"]}, + "GPL-3.0-only": {"has": ["gnu general public license", "version 3, 29 june 2007"]}, + "LLVM-exception": {"has": ["llvm exceptions to the apache 2.0 license"]}, +} +CRATES_IO = ("registry+https://github.com/rust-lang/crates.io-index", "sparse+https://index.crates.io/") +TREE_LINE = re.compile(r"^(\S+) v(\S+)(?: \((.+)\))?$") +GITHUB_REPO = re.compile(r"github\.com[/:]([^/]+)/([^/#?]+?)(?:\.git)?(?:[/#?].*)?$") +SPDX_TOKEN = re.compile(r"\(|\)|[^\s()]+") +RULE = "=" * 80 +THIN_RULE = "-" * 80 + + +class Gap(Exception): + """A missing or ambiguous input that would make the notices incomplete.""" + + +def run(cmd): + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise Gap(f"{' '.join(cmd)} failed:\n{result.stderr}") + return result.stdout + + +def read_bytes(path): + with open(path, "rb") as handle: + return handle.read() + + +def decode(data, where): + try: + return data.decode("utf-8") + except UnicodeDecodeError as error: + raise Gap(f"{where} is not UTF-8 ({error}); add it by hand") from error + + +# --- dependency graph ------------------------------------------------------- + +def cargo_selection(args): + # Colour is off so CARGO_TERM_COLOR=always in CI cannot corrupt the output. + selection = ["--manifest-path", args.manifest_path, "--locked", "--color", "never"] + if args.package: + selection += ["-p", args.package] + if args.no_default_features: + selection.append("--no-default-features") + if args.features: + selection += ["--features", args.features] + return selection + + +def release_graph(args): + """(name, version, annotation) of every package in the build, per cargo tree.""" + nodes = set() + for target in args.target: + out = run(["cargo", "tree", "-e", "normal,build", "--target", target, + "--prefix", "none", "--format", "{p}"] + cargo_selection(args)) + for raw in out.splitlines(): + line = raw.strip().removesuffix(" (*)").removesuffix(" (proc-macro)") + if not line: + continue + match = TREE_LINE.match(line) + if not match: + raise Gap(f"cannot parse cargo tree line: {raw!r}") + nodes.add((match.group(1), match.group(2), match.group(3) or "")) + return nodes + + +def resolve_package(name, version, annotation, by_name_version): + """Match a cargo tree node to exactly one cargo metadata package.""" + candidates = by_name_version.get((name, version), []) + if not annotation: + matches = [p for p in candidates if (p["source"] or "").startswith(CRATES_IO)] + elif annotation.startswith("registry "): + raise Gap(f"{name} {version} comes from {annotation}; only crates.io is supported") + elif "://" in annotation: + matches = [p for p in candidates if (p["source"] or "").startswith("git+" + annotation)] + else: + wanted = os.path.realpath(annotation) + matches = [p for p in candidates if p["source"] is None + and os.path.realpath(os.path.dirname(p["manifest_path"])) == wanted] + if len(matches) != 1: + raise Gap(f"{name} {version} ({annotation or 'crates.io'}) matches " + f"{len(matches)} packages in cargo metadata") + return matches[0] + + +def source_url(package): + source = package["source"] or "" + name, version = package["name"], package["version"] + if source.startswith(CRATES_IO): + return f"https://static.crates.io/crates/{name}/{name}-{version}.crate" + if source.startswith("git+"): + url, _, rev = source[4:].partition("#") + return f"{url.split('?')[0]} at commit {rev}" + return os.path.dirname(package["manifest_path"]) + + +# --- licence policy --------------------------------------------------------- + +def parse_spdx(expression): + """Parse an SPDX expression (plus the legacy `/` for OR) into a tree.""" + tokens = SPDX_TOKEN.findall(expression.replace("/", " OR ")) + position = 0 + + def peek(): + return tokens[position] if position < len(tokens) else None + + def take(): + nonlocal position + if position >= len(tokens): + raise Gap(f"malformed licence expression {expression!r}") + position += 1 + return tokens[position - 1] + + def parse_or(): + node = parse_and() + while peek() == "OR": + take() + node = ("OR", node, parse_and()) + return node + + def parse_and(): + node = parse_atom() + while peek() == "AND": + take() + node = ("AND", node, parse_atom()) + return node + + def parse_atom(): + token = take() + if token == "(": + node = parse_or() + if take() != ")": + raise Gap(f"unbalanced licence expression {expression!r}") + return node + if token in (")", "AND", "OR", "WITH"): + raise Gap(f"malformed licence expression {expression!r}") + if peek() == "WITH": + take() + return ("LEAF", f"{token} WITH {take()}") + return ("LEAF", token) + + tree = parse_or() + if peek() is not None: + raise Gap(f"malformed licence expression {expression!r}") + return tree + + +def branches(tree): + """The expression as alternatives, each a tuple of licences that all apply.""" + kind = tree[0] + if kind == "LEAF": + return [(tree[1],)] + left, right = branches(tree[1]), branches(tree[2]) + if kind == "OR": + return left + right + return [a + b for a in left for b in right] + + +def identifies(licence, text): + markers = LICENCE_MARKERS.get(licence) + if markers is None: + return False + return (all(phrase in text for phrase in markers["has"]) + and not any(phrase in text for phrase in markers.get("lacks", []))) + + +COMMENT_PREFIX = re.compile(r"^\s*(//+|#+|\*+|/\*+|--|;+)?\s?", re.M) + + +def normalise(text): + """Lower-cased text with comment prefixes removed and whitespace collapsed, + so licence texts written as source comments are still recognised.""" + return re.sub(r"\s+", " ", COMMENT_PREFIX.sub("", text).lower()) + + +def has_text(licence, texts): + """Whether some text identifies the licence; `X WITH Y` needs both parts.""" + parts = licence.split(" WITH ") + return all(any(identifies(part, text) for text in texts) for part in parts) + + +def elect(declared, rank, texts): + """The licence terms we redistribute under: the most preferred allowed + alternative whose every licence text is among the crate-level files.""" + normalised = [normalise(text) for text in texts] + allowed = sorted((max(rank[licence] for licence in branch), branch) + for branch in branches(parse_spdx(declared)) + if all(licence in rank for licence in branch)) + if not allowed: + return None, "the policy allows none of its licences" + for _, branch in allowed: + if all(has_text(licence, normalised) for licence in branch): + return " AND ".join(dict.fromkeys(branch)), None + return None, "no crate-level licence file contains the text of an allowed option" + + +# --- licence texts ---------------------------------------------------------- + +def crate_licence_files(package): + """Licence-like files anywhere in the crate, plus its declared license-file, + as (label, bytes, governs). A file governs the crate itself when it sits at + the crate root, under a top-level LICENSES directory, or is the declared + license-file; deeper files belong to bundled components.""" + root = os.path.dirname(package["manifest_path"]) + found = {} + for directory, subdirs, files in os.walk(root): + subdirs[:] = sorted(d for d in subdirs if d != "target") + parts = [] if directory == root else os.path.relpath(directory, root).split(os.sep) + inside_licenses = "licenses" in (part.lower() for part in parts) + governs = not parts or parts[0].lower() == "licenses" + for name in sorted(files): + if inside_licenses or LICENCE_FILE.match(name): + path = os.path.join(directory, name) + found[os.path.relpath(path, root)] = (path, governs) + declared = package.get("license_file") + if declared: + path = os.path.normpath(os.path.join(root, declared)) + if not os.path.isfile(path): + raise Gap(f"{package['name']} {package['version']} declares license-file " + f"{declared!r}, which is missing") + found[os.path.relpath(path, root)] = (path, True) + return [(label, read_bytes(path), governs) for label, (path, governs) in sorted(found.items())] + + +def checkout_licence_files(package): + """For a git dependency, licence files from the crate directory up to the + root of Cargo's checkout of that repository at the locked commit.""" + directory = os.path.realpath(os.path.dirname(package["manifest_path"])) + files = [] + while True: + for name in sorted(os.listdir(directory)): + path = os.path.join(directory, name) + if os.path.isfile(path) and LICENCE_FILE.match(name): + files.append((path, read_bytes(path))) + elif os.path.isdir(path) and name.lower() == "licenses": + for inner, _, inner_files in os.walk(path): + files += [(os.path.join(inner, f), read_bytes(os.path.join(inner, f))) + for f in sorted(inner_files)] + if os.path.exists(os.path.join(directory, ".cargo-ok")) or os.path.exists(os.path.join(directory, ".git")): + root = directory + break + parent = os.path.dirname(directory) + if parent == directory: + return [] + directory = parent + rev = (package["source"] or "").partition("#")[2] + return [(f"{os.path.relpath(path, root)} at {rev[:12]}", data, True) for path, data in files] + + +def ships_native_files(package): + root = os.path.dirname(package["manifest_path"]) + for _, subdirs, files in os.walk(root): + subdirs[:] = [d for d in subdirs if d != "target"] + if any(NATIVE_FILE.search(name) for name in files): + return True + return False + + +def bundled_files(package, patterns, governs=False): + """Extra licence files a config entry names: by default for bundled code, + or, with `files_govern = true`, the crate's own unusually named licence.""" + root = os.path.realpath(os.path.dirname(package["manifest_path"])) + files = [] + for pattern in patterns: + matches = sorted(p for p in glob.glob(os.path.join(root, pattern), recursive=True) + if os.path.isfile(p)) + if not matches: + raise Gap(f"{package['name']} {package['version']}: bundled licence file " + f"{pattern!r} not found") + for path in matches: + if os.path.commonpath([root, os.path.realpath(path)]) != root: + raise Gap(f"{package['name']}: {pattern!r} escapes the crate") + files.append((os.path.relpath(path, root), read_bytes(path), governs)) + return files + + +class GitHub: + def __init__(self): + self.token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + self.cache = {} + + def get(self, url, raw=False): + if url in self.cache: + return self.cache[url] + headers = {"User-Agent": "third-party-notices"} + if self.token and url.startswith("https://api.github.com/"): + headers["Authorization"] = f"Bearer {self.token}" + last_error = None + for attempt in range(6): + try: + request = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(request, timeout=30) as response: + body = response.read() + value = body if raw else json.loads(body) + self.cache[url] = value + return value + except urllib.error.HTTPError as error: + if error.code == 404: + self.cache[url] = None + return None + last_error = error + except (urllib.error.URLError, http.client.HTTPException, OSError) as error: + # Covers timeouts and dropped connections as well as DNS errors. + last_error = error + time.sleep(2 ** attempt) + raise Gap(f"request failed after retries: {url}: {last_error}") + + def licence_files(self, package): + """Licence-like files from the crate's directory up to its repository + root, at the commit the crate was published from.""" + match = GITHUB_REPO.search(package.get("repository") or "") + vcs_path = os.path.join(os.path.dirname(package["manifest_path"]), ".cargo_vcs_info.json") + if not match or not os.path.isfile(vcs_path): + return [] + owner, repo = match.group(1), match.group(2) + with open(vcs_path) as handle: + vcs = json.load(handle) + sha = vcs.get("git", {}).get("sha1") + if not sha: + return [] + files = [] + directory = vcs.get("path_in_vcs", "") + while True: + for entry in self.listing(owner, repo, sha, directory): + if entry["type"] == "file" and LICENCE_FILE.match(entry["name"]): + files.append(entry) + elif entry["type"] == "dir" and entry["name"].lower() == "licenses": + files += self.all_files(owner, repo, sha, entry["path"]) + if not directory: + break + directory = os.path.dirname(directory) + where = f"{owner}/{repo}@{sha[:12]}" + return [(f"{where}/{entry['path']}", self.get(entry["download_url"], raw=True), True) + for entry in sorted(files, key=lambda entry: entry["path"])] + + def all_files(self, owner, repo, sha, directory): + files = [] + for entry in self.listing(owner, repo, sha, directory): + if entry["type"] == "file": + files.append(entry) + elif entry["type"] == "dir": + files += self.all_files(owner, repo, sha, entry["path"]) + return files + + def listing(self, owner, repo, sha, directory): + url = f"https://api.github.com/repos/{owner}/{repo}/contents/{directory}?ref={sha}" + listing = self.get(url) + return sorted(listing, key=lambda entry: entry["name"]) if isinstance(listing, list) else [] + + +# --- generation ------------------------------------------------------------- + +def entry_for(table, package, required_version=False): + """The config entry for a package. Entries may name a `version`; exception + and missing-text entries must, because their conclusions are per release.""" + entry = table.get(package["name"]) + if entry is None: + return None + if "version" in entry: + return entry if entry["version"] == package["version"] else None + if required_version: + raise Gap(f"config entry for {package['name']} must name the reviewed `version`") + return entry + + +def generate(args, config, github): + rank = {licence: index for index, licence in enumerate(config["allowed"])} + exceptions = config.get("exception", {}) + native_config = config.get("native", {}) + remarks = config.get("remark", {}) + missing_config = config.get("missing", {}) + used = set() + + # The same feature selection as the graph, so packages that only the + # requested features pull in are present. + selection = ["--no-default-features"] if args.no_default_features else [] + if args.features: + selection += ["--features", args.features] + metadata = json.loads(run(["cargo", "metadata", "--format-version", "1", "--color", "never", + "--manifest-path", args.manifest_path, "--locked"] + selection)) + by_name_version = {} + for package in metadata["packages"]: + by_name_version.setdefault((package["name"], package["version"]), []).append(package) + workspace = set(metadata["workspace_members"]) + + gaps, index, notes, texts = [], [], [], {} + for name, version, annotation in sorted(release_graph(args)): + crate = f"{name} {version}" + try: + package = resolve_package(name, version, annotation, by_name_version) + if package["id"] in workspace: + continue # first-party code, covered by the project's own licence + + declared = package.get("license") + files = crate_licence_files(package) + origins = {"crate"} if files else set() + + def repository_files(): + if (package["source"] or "").startswith("git+"): + return checkout_licence_files(package) + return github.licence_files(package) + + if not files: + files = repository_files() + origins = {"repository"} if files else set() + + # Code the crate bundles: its licence files are reproduced, but they + # never stand in for the crate's own licence. + build_deps = {dep["name"] for dep in package["dependencies"] if dep.get("kind") == "build"} + builds_native = bool(build_deps & NATIVE_BUILD_DEPS or package.get("links") + or name.endswith("-src") or ships_native_files(package)) + native = entry_for(native_config, package) if builds_native else None + if builds_native and native is None: + raise Gap(f"{crate} may contain native code; review what it bundles and " + f"add [native.{name}] to the config") + if native is not None: + used.add(("native", name)) + labels = {label for label, _, _ in files} + files += [extra for extra in bundled_files(package, native.get("files", []), + native.get("files_govern", False)) + if extra[0] not in labels] + + exception = entry_for(exceptions, package, required_version=True) + missing = entry_for(missing_config, package, required_version=True) + if exception is None and not declared: + raise Gap(f"{crate} declares no SPDX licence; review it and add [exception.{name}]") + + def decide(): + governing = [decode(data, f"{crate}: {label}") for label, data, governs in files if governs] + if exception is None: + return elect(declared, rank, governing) + licence = exception["licence"] + if declared and (licence,) not in branches(parse_spdx(declared)): + raise Gap(f"{crate}: [exception.{name}] licence {licence!r} is not one " + f"of {declared!r}") + if has_text(licence, [normalise(text) for text in governing]): + return licence, None + return None, f"no crate-level licence file contains the {licence} text" + + # When the chosen licence's text is not among the crate-level files, + # look in its repository, then in a reviewed supplement from the config. + elected, problem = decide() + if elected is None and "repository" not in origins: + known = {hashlib.sha256(data).hexdigest() for _, data, governs in files if governs} + extra = [item for item in repository_files() + if hashlib.sha256(item[1]).hexdigest() not in known] + if extra: + files += extra + origins.add("repository") + elected, problem = decide() + if elected is None and missing is not None: + used.add(("missing", name)) + text = github.get(missing["text_url"], raw=True) + if text is None or hashlib.sha256(text).hexdigest() != missing["text_sha256"]: + raise Gap(f"{crate}: {missing['text_url']} does not match text_sha256") + files.append((missing["text_url"], text, True)) + origins.add("config") + notes.append((crate, missing["note"])) + elected, problem = decide() + if elected is None: + raise Gap(f"{crate} ({declared or 'no licence declared'}): {problem}; review it " + f"and add [missing.{name}] or [exception.{name}]") + if exception is not None: + used.add(("exception", name)) + notes.append((crate, exception["reason"])) + if native is not None: + notes.append((crate, native["note"])) + remark = entry_for(remarks, package) + if remark is not None: + used.add(("remark", name)) + notes.append((crate, remark["note"])) + + ids = [] + for label, data, _ in files: + digest = hashlib.sha256(data).hexdigest() + if digest not in texts: + texts[digest] = {"text": decode(data, f"{crate}: {label}"), "shipped_by": []} + texts[digest]["shipped_by"].append((crate, label)) + ids.append(digest) + except Gap as gap: + gaps.append(str(gap)) + continue + index.append((crate, declared or "(none declared)", elected, source_url(package), origins, ids)) + + for kind, table in (("native", native_config), ("missing", missing_config), + ("exception", exceptions), ("remark", remarks)): + for name in sorted(table): + if (kind, name) not in used: + print(f"note: [{kind}.{name}] is not used by this build", file=sys.stderr) + if gaps: + raise Gap("cannot produce complete notices:\n " + "\n ".join(gaps)) + return index, notes, texts + + +def musl_copyright(github): + """musl's COPYRIGHT for the release the active rustc's musl targets use. + + Rust records the musl version it builds those targets from in + src/ci/docker/scripts/musl.sh; the COPYRIGHT comes from that musl release + tarball.""" + info = run(["rustc", "-vV"]) + commit = re.search(r"^commit-hash: (\S+)$", info, re.M) + release = re.search(r"^release: (\S+)$", info, re.M) + if not commit or not release: + raise Gap(f"cannot read rustc's commit hash from:\n{info}") + script = github.get("https://raw.githubusercontent.com/rust-lang/rust/" + f"{commit.group(1)}/src/ci/docker/scripts/musl.sh", raw=True) + match = re.search(rb"^MUSL=musl-(\S+)$", script or b"", re.M) + if not match: + raise Gap(f"cannot find the musl version for rustc {release.group(1)}") + version = match.group(1).decode() + archive = github.get(f"https://musl.libc.org/releases/musl-{version}.tar.gz", raw=True) + if archive is None: + raise Gap(f"musl {version} release tarball not found") + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar: + member = tar.extractfile(f"musl-{version}/COPYRIGHT") + if member is None: + raise Gap(f"musl {version} tarball has no COPYRIGHT") + text = decode(member.read(), f"musl {version} COPYRIGHT") + return version, f"Rust {release.group(1)}", text + + +def render(args, config, config_dir, index, notes, texts, github): + numbers = {} + for digest, _ in sorted(texts.items(), key=lambda item: item[1]["shipped_by"][0]): + numbers[digest] = len(numbers) + 1 + if args.no_default_features: + features = f"{args.features or 'none'} (default features off)" + else: + features = "default" + (f" plus {args.features}" if args.features else "") + + out = ["THIRD-PARTY SOFTWARE NOTICES", "", config["header"].strip(), ""] + out += [ + "The crates below are compiled into this build or used to build it, unmodified", + "from the source each entry names. Every licence, copyright, notice and", + "authors file they ship is reproduced unaltered in the TEXTS section,", + "identical texts once. Where a crate offers a choice of licences, it is", + "redistributed under the terms named. Each Source link is the exact,", + "unmodified source of that crate as used for this build, which is also how", + "source is made available where a licence requires it (for example MPL-2.0).", + "", + f"Targets: {', '.join(args.target)}", + f"Features: {features}", + "", + RULE, "CRATES", RULE, + ] + for crate, declared, elected, url, origins, ids in index: + refs = ", ".join(f"[{numbers[digest]}]" for digest in dict.fromkeys(ids)) + where = "" + if "repository" in origins: + where += " (from the crate's repository)" if origins == {"repository"} else \ + " (some from the crate's repository)" + if "config" in origins: + where += " (see NOTES)" + out.append(crate) + out.append(f" Licence: {declared}") + if elected != declared: + out.append(f" Redistributed under: {elected}") + out.append(f" Source: {url}") + out.append(f" Texts: {refs}{where}") + if notes: + out += ["", RULE, "NOTES", RULE] + out += [f"{crate}: {note}" for crate, note in notes] + out += ["", RULE, "TEXTS", RULE] + for digest, number in numbers.items(): + entry = texts[digest] + shipped = ", ".join(f"{crate} ({label})" for crate, label in entry["shipped_by"]) + out += ["", THIN_RULE, f"[{number}] Shipped by: {shipped}", THIN_RULE, "", + framed(entry["text"])] + for appendix in config.get("appendix", []): + wanted = appendix.get("targets") + if wanted and not set(wanted) & set(args.target): + continue + out += ["", RULE, appendix["title"].upper(), RULE, "", appendix["intro"].strip()] + if "file" in appendix: + path = os.path.join(config_dir, appendix["file"]) + out += ["", framed(decode(read_bytes(path), path))] + if appendix.get("musl_from_rustc"): + version, rust, text = musl_copyright(github) + out += ["", f"musl {version}, the release {rust} builds its musl targets from:", + "", framed(text)] + return "\n".join(out) + "\n" + + +def framed(text): + """A text exactly as shipped, minus the final newline the join adds back.""" + return text[:-1] if text.endswith("\n") else text + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--manifest-path", default="Cargo.toml") + parser.add_argument("--package") + parser.add_argument("--features") + parser.add_argument("--no-default-features", action="store_true") + parser.add_argument("--target", action="append", required=True) + parser.add_argument("--config", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + config_dir = os.path.dirname(os.path.abspath(args.config)) + with open(args.config, "rb") as handle: + config = tomllib.load(handle) + try: + github = GitHub() + index, notes, texts = generate(args, config, github) + document = render(args, config, config_dir, index, notes, texts, github) + except Gap as gap: + sys.exit(f"error: {gap}") + with open(args.output, "w", encoding="utf-8", newline="\n") as handle: + handle.write(document) + print(f"{len(index)} crates, {len(texts)} distinct texts -> {args.output}", file=sys.stderr) + + +if __name__ == "__main__": + main()