From 0a34bce5add9d56f9ba9e218889889d05749d8ed Mon Sep 17 00:00:00 2001 From: Eusebio Resende Date: Thu, 22 Oct 2026 17:19:27 +0100 Subject: [PATCH 1/3] Build and test platform wheels with release artifact bundles --- .circleci/config.yml | 24 +++++- .github/workflows/wheels.yml | 121 ++++++++++++++++++++++++++ CONTRIBUTING.rst | 91 ++++++++++++++++++++ MANIFEST.in | 4 +- ci/release_bundle.py | 162 +++++++++++++++++++++++++++++++++++ ci/test_release_bundle.py | 143 +++++++++++++++++++++++++++++++ pyproject.toml | 20 ++++- setup.py | 1 + tests/conftest.py | 3 + tests/nuodb_cython_test.py | 1 + 10 files changed, 561 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/wheels.yml create mode 100644 ci/release_bundle.py create mode 100644 ci/test_release_bundle.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 1701596..1cc98ef 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,8 +55,13 @@ jobs: name: Make test directories command: mkdir -p artifacts results - run: - name: Install Python dependencies - command: make PYTHON=python3 install + name: Build, validate, and install the regression wheel + command: | + python3 -m pip install 'build>=1.2,<2' 'twine>=6,<8' -r test_requirements.txt + # The default build creates an sdist, then builds the wheel from it. + python3 -m build --outdir artifacts/dist + python3 -m twine check --strict artifacts/dist/* + python3 -m pip install --force-reinstall artifacts/dist/*.whl - run: name: Start NuoDB Admin command: | @@ -65,8 +70,19 @@ jobs: sudo -u nuodb /opt/nuodb/etc/nuoadmin start sudo -u nuodb /opt/nuodb/bin/nuocmd --show-json get effective-license - run: - name: Run test - command: make PYTHON=python3 fulltest + name: Run regressions against the installed wheel + command: | + PROJECT_DIR="$PWD" + TEST_DIR=$(mktemp -d) + trap 'rm -rf "$TEST_DIR"' EXIT + cp -a tests "$TEST_DIR/" + cd "$TEST_DIR" + # A failed optional extension build must not pass release testing. + python3 -c 'import pynuodb; import pynuodb._fetch; print(pynuodb.__file__)' + PATH="/opt/nuodb/bin:$PATH" python3 -m pytest tests \ + --show-capture=stdout \ + --junitxml="$PROJECT_DIR/results/result.xml" \ + --log-file="$PROJECT_DIR/artifacts/testlog.out" --log-file-level=INFO - store_artifacts: path: artifacts - store_test_results: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..663b898 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,121 @@ +name: Build distribution artifacts + +on: + pull_request: + push: + branches: [master] + tags: ['v*'] + workflow_dispatch: + +# No publishing credentials or OIDC permissions belong in this workflow. +permissions: + contents: read + +jobs: + sdist: + name: Build source distribution + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.11' + - name: Install release tooling + run: python -m pip install build==1.6.1 twine==7.0.0 packaging 'pytest>=8,<9' + - name: Test release-bundle validation + run: python -m pytest -v ci/test_release_bundle.py + - name: Build and check source distribution + run: | + python -m build --sdist + python -m twine check --strict dist/* + - name: Check source installation without a compiler + shell: bash + run: | + CC=/no-c-compiler python -m pip install --no-cache-dir dist/*.tar.gz + cd "$RUNNER_TEMP" + python -c 'import pynuodb; from pynuodb import encodedsession; assert not encodedsession._HAVE_FETCH_ACCEL' + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: sdist + path: dist/*.tar.gz + if-no-files-found: error + retention-days: 90 + + wheels: + name: Wheels (${{ matrix.target }}) + needs: sdist + strategy: + fail-fast: false + matrix: + include: + - target: linux-x86_64 + os: ubuntu-24.04 + arch: x86_64 + - target: linux-aarch64 + os: ubuntu-24.04-arm + arch: aarch64 + - target: macos-x86_64 + os: macos-15-intel + arch: x86_64 + - target: macos-arm64 + os: macos-15 + arch: arm64 + - target: windows-amd64 + os: windows-2022 + arch: AMD64 + runs-on: ${{ matrix.os }} + steps: + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.11' + - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 + with: + name: sdist + path: dist + - name: Install wheel tooling + run: python -m pip install cibuildwheel==4.3.4 + - name: Build and test wheels from the source distribution + shell: bash + env: + CIBW_ARCHS: ${{ matrix.arch }} + run: python -m cibuildwheel --output-dir wheelhouse dist/*.tar.gz + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: wheels-${{ matrix.target }} + path: wheelhouse/*.whl + if-no-files-found: error + retention-days: 90 + + release-bundle: + name: Validate and assemble release bundle + needs: [sdist, wheels] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.11' + - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 + with: + name: sdist + path: release-bundle/dist + - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 + with: + pattern: wheels-* + merge-multiple: true + path: release-bundle/dist + - name: Validate distributions and record provenance + run: | + python -m pip install twine==7.0.0 packaging + python -m twine check --strict release-bundle/dist/* + python ci/release_bundle.py release-bundle + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: release-bundle-${{ github.run_id }}-${{ github.run_attempt }} + path: release-bundle/ + if-no-files-found: error + retention-days: 90 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index cc340b9..6c9cdfb 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -77,3 +77,94 @@ Release ------- Maintain the list of changes per release in CHANGES.rst. Also note the known defects. + +Building distribution artifacts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``.github/workflows/wheels.yml`` builds distribution artifacts on pull requests, +pushes to ``master``, version tags matching ``v*``, and manual workflow runs. +It does **not** publish to PyPI, request an OIDC token, or use PyPI credentials. +The internal deployment process remains responsible for authorizing a release +and publishing it. + +The initial wheel matrix is CPython 3.9 through 3.13 (standard, GIL-enabled +builds) on: + +* Linux x86-64 and ARM64, using ``manylinux2014`` (glibc >= 2.17). +* macOS Intel and Apple Silicon, using separate architecture-specific wheels. +* Windows x86-64. + +PyPy, free-threaded Python, 32-bit targets, Windows ARM64, and Alpine/musl +wheels are not included. A source distribution is provided for other systems; +its extension build is optional, allowing the pure-Python fallback when a C +compiler is unavailable. The workflow also checks that no-compiler source +installation path. Package dependencies must still be installable. + +The source archive is built once. Every platform builds its wheels from that +same archive, which includes the Cython sources, generated C, headers, and test +sources. ``cibuildwheel`` installs each wheel into an isolated test environment +and runs the database-independent accelerator, encoding, and date/time tests. +The tests import the installed wheel, not the source checkout, and fail if the +compiled extension is missing. Windows tests install ``tzdata`` for IANA time +zones. + +CircleCI continues to run the full database regression suite and performance +comparison. Its regression job now builds and installs a wheel from an sdist +and runs the tests outside the checkout. The job saves its native wheel and +source archive under ``artifacts/dist`` for diagnosis; that native Linux wheel +is **not** a portable manylinux release artifact. + +Release bundle +^^^^^^^^^^^^^^ + +Only after all GitHub wheel jobs pass does the workflow assemble the artifact +``release-bundle--``. It contains:: + + dist/ + pynuodb-.tar.gz + pynuodb----.whl + ... + manifest.json + SHA256SUMS + +``ci/release_bundle.py`` requires exactly one source archive and one compiled +wheel for each of the 25 Python/platform combinations. It checks package names, +versions, wheel ABIs/platforms, extension presence, and required source-archive +contents. On tag runs, the tag version must match the package version +(``v5.0`` and ``v5.0.0`` are equivalent). ``twine check --strict`` checks all +distribution metadata. + +The manifest records distribution sizes and SHA-256 checksums, the source +commit/ref, repository, workflow ref, event, run ID, and run attempt. Artifacts +request 90-day retention (subject to repository/organization limits); archive +approved bundles internally before they expire. Dependency resolution and +builder images can change, so a rebuild is not guaranteed to be byte-identical. + +For internal publishing: + +#. Select a successful run of this workflow for an approved release commit or + protected version tag. Use its exact run ID and attempt, not "latest". +#. Confirm that CircleCI database regressions passed for the same release + commit. GitHub's bundle job does not automatically gate on CircleCI status. +#. Download the **final release bundle**, not the intermediate wheel artifacts + or CircleCI's diagnostic distributions. Check its manifest against the + approved repository, commit, version, and run identity. Pull-request runs + build a merge commit for testing and must not be used for publication. +#. Verify the downloaded distributions from the bundle directory:: + + sha256sum --check SHA256SUMS + +#. Preserve the approved bundle internally and pass only ``dist/*`` to the + separate publishing job. Do not rebuild distributions during publishing. + +Checksums detect corruption; they do not authenticate an untrusted build or +replace release approval. No GitHub-to-internal-GitLab connection is required: +the internal runner can fetch the selected artifact using outbound access. +The PyPI token must never be supplied to this workflow or the wheel tests. + +When adding Python versions or targets, update ``pyproject.toml``, the workflow +matrix, and ``PYTHONS``/``TARGETS`` in ``ci/release_bundle.py`` together. Test the +bundle validator locally with:: + + python -m pip install packaging 'pytest>=8,<9' + python -m pytest ci/test_release_bundle.py diff --git a/MANIFEST.in b/MANIFEST.in index f3ecc96..ff8e22b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,4 @@ include README.rst LICENSE -recursive-include pynuodb *.pyx *.pxd *.pxi *.c *.h \ No newline at end of file +recursive-include pynuodb *.pyx *.pxd *.pxi *.c *.h +# cibuildwheel builds and tests from this same source archive on every platform. +recursive-include tests *.py diff --git a/ci/release_bundle.py b/ci/release_bundle.py new file mode 100644 index 0000000..eaa5a87 --- /dev/null +++ b/ci/release_bundle.py @@ -0,0 +1,162 @@ +"""Validate the complete distribution set and write its release manifest. + +This does not upload anything or execute code from a distribution. Checksums +provide an integrity check, not a signature or authorization to publish. +""" + +import argparse +from email.parser import BytesParser +import hashlib +import json +import os +from pathlib import Path +import tarfile +import zipfile + +from packaging.utils import parse_sdist_filename, parse_wheel_filename +from packaging.version import Version + + +# Keep in sync with pyproject.toml and .github/workflows/wheels.yml. +PYTHONS = ("cp39", "cp310", "cp311", "cp312", "cp313") +TARGETS = ("linux-x86_64", "linux-aarch64", "macos-x86_64", + "macos-arm64", "windows-amd64") +SOURCE_FILES = { + "pyproject.toml", "setup.py", "README.rst", "LICENSE", + "pynuodb/_fetch.pyx", "pynuodb/_fetch.c", "pynuodb/_cutil.h", + "pynuodb/_codes.pxi", "pynuodb/_resultset.pxi", + "tests/__init__.py", "tests/conftest.py", "tests/nuodb_base.py", + "tests/nuodb_cython_test.py", "tests/nuodb_crypt_test.py", + "tests/nuodb_datatype_ticks_test.py", +} + + +def package_version(data): + metadata = BytesParser().parsebytes(data) + if metadata["Name"] != "pynuodb" or not metadata["Version"]: + raise ValueError("Distribution metadata must identify pynuodb and its version") + return Version(metadata["Version"]) + + +def wheel_target(platform): + for arch in ("x86_64", "aarch64"): + if platform in ("manylinux2014_" + arch, "manylinux_2_17_" + arch): + return "linux-" + arch + for arch in ("x86_64", "arm64"): + if platform.startswith("macosx_") and platform.endswith("_" + arch): + return "macos-" + arch + if platform == "win_amd64": + return "windows-amd64" + raise ValueError("Unexpected wheel platform: " + platform) + + +def validate_sdist(path): + name, version = parse_sdist_filename(path.name) + if name != "pynuodb": + raise ValueError("Unexpected source distribution: " + path.name) + with tarfile.open(path) as archive: + files = {member.name: member for member in archive.getmembers() if member.isfile()} + metadata = [name for name in files + if name.endswith("/PKG-INFO") and name.count("/") == 1] + if len(metadata) != 1: + raise ValueError("Source distribution must contain one top-level PKG-INFO") + root = metadata[0].split("/")[0] + missing = SOURCE_FILES - {name[len(root) + 1:] for name in files + if name.startswith(root + "/")} + if missing: + raise ValueError("Source distribution is missing: " + ", ".join(sorted(missing))) + with archive.extractfile(files[metadata[0]]) as stream: + if package_version(stream.read()) != version: + raise ValueError("Source distribution filename/metadata version mismatch") + return version + + +def validate_wheel(path, version): + name, file_version, _, tags = parse_wheel_filename(path.name) + if name != "pynuodb" or file_version != version: + raise ValueError("Wheel name/version does not match source distribution: " + path.name) + targets = set() + for tag in tags: + if tag.interpreter not in PYTHONS or tag.abi != tag.interpreter: + raise ValueError("Unexpected Python ABI: " + str(tag)) + targets.add((tag.interpreter, wheel_target(tag.platform))) + if len(targets) != 1: + raise ValueError("Wheel must cover exactly one Python/target pair: " + path.name) + with zipfile.ZipFile(path) as archive: + names = archive.namelist() + metadata = [name for name in names if name.endswith(".dist-info/METADATA")] + if len(metadata) != 1 or package_version(archive.read(metadata[0])) != version: + raise ValueError("Wheel metadata mismatch: " + path.name) + if not any(name.startswith("pynuodb/_fetch.") and name.endswith((".so", ".pyd")) + for name in names): + raise ValueError("Wheel has no compiled _fetch extension: " + path.name) + return targets.pop() + + +def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def build_manifest(bundle, provenance): + dist = bundle / "dist" + files = sorted(dist.iterdir()) + if any(path.is_symlink() or not path.is_file() for path in files): + raise ValueError("dist/ must contain only regular distribution files") + sdists = [path for path in files if path.name.endswith(".tar.gz")] + wheels = [path for path in files if path.suffix == ".whl"] + if len(sdists) != 1 or len(files) != len(sdists) + len(wheels): + raise ValueError("Expected one source archive and only wheel files in dist/") + version = validate_sdist(sdists[0]) + ref = provenance["ref"] + if ref.startswith("refs/tags/"): + if Version(ref.removeprefix("refs/tags/").removeprefix("v")) != version: + raise ValueError("Release tag does not match distribution version") + + expected = {(python, target) for python in PYTHONS for target in TARGETS} + found = set() + for wheel in wheels: + pair = validate_wheel(wheel, version) + if pair in found: + raise ValueError("Duplicate wheel coverage: " + str(pair)) + found.add(pair) + if found != expected: + raise ValueError("Incomplete wheel matrix; missing: " + str(sorted(expected - found))) + + return { + "schema_version": 1, + "package": "pynuodb", + "version": str(version), + "source": provenance, + "files": [ + {"filename": path.name, "size": path.stat().st_size, "sha256": sha256(path)} + for path in files + ], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("bundle", type=Path, help="directory containing dist/") + args = parser.parse_args() + provenance = {key: os.environ[env] for key, env in { + "repository": "GITHUB_REPOSITORY", "commit": "GITHUB_SHA", "ref": "GITHUB_REF", + "event": "GITHUB_EVENT_NAME", "run_id": "GITHUB_RUN_ID", + "run_attempt": "GITHUB_RUN_ATTEMPT", "workflow_ref": "GITHUB_WORKFLOW_REF", + }.items()} + provenance["run_url"] = "{}/{}/actions/runs/{}".format( + os.environ["GITHUB_SERVER_URL"], provenance["repository"], provenance["run_id"]) + manifest = build_manifest(args.bundle, provenance) + (args.bundle / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + checksums = "".join("{} dist/{}\n".format(file["sha256"], file["filename"]) + for file in manifest["files"]) + (args.bundle / "SHA256SUMS").write_text(checksums) + print("Validated {} distributions for pynuodb {}".format( + len(manifest["files"]), manifest["version"])) + + +if __name__ == "__main__": + main() diff --git a/ci/test_release_bundle.py b/ci/test_release_bundle.py new file mode 100644 index 0000000..e10e3b4 --- /dev/null +++ b/ci/test_release_bundle.py @@ -0,0 +1,143 @@ +"""Tests for release validation, independent of a compiler or NuoDB server.""" + +import hashlib +import io +import json +import tarfile +import zipfile + +import pytest + +from ci import release_bundle as release + + +VERSION = "5.0" +PROVENANCE = {"ref": "refs/tags/v5.0", "commit": "a" * 40, "run_id": "123"} + + +def metadata(version=VERSION): + return ("Metadata-Version: 2.1\nName: pynuodb\nVersion: " + version + "\n").encode() + + +def write_sdist(dist, missing=None): + path = dist / ("pynuodb-" + VERSION + ".tar.gz") + with tarfile.open(path, "w:gz") as archive: + files = {name: b"placeholder\n" for name in release.SOURCE_FILES if name != missing} + files["PKG-INFO"] = metadata() + for name, data in files.items(): + member = tarfile.TarInfo("pynuodb-" + VERSION + "/" + name) + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + return path + + +def write_wheel(dist, python="cp39", platform="win_amd64", version=VERSION, + compiled=True, metadata_version=VERSION, abi=None): + filename = "pynuodb-{}-{}-{}-{}.whl".format(version, python, abi or python, platform) + path = dist / filename + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("pynuodb-{}.dist-info/METADATA".format(version), + metadata(metadata_version)) + if compiled: + extension = ".pyd" if platform == "win_amd64" else ".so" + archive.writestr("pynuodb/_fetch." + python + extension, b"binary placeholder") + return path + + +@pytest.fixture +def bundle(tmp_path): + dist = tmp_path / "dist" + dist.mkdir() + write_sdist(dist) + for python in release.PYTHONS: + for platform in ("manylinux_2_17_x86_64.manylinux2014_x86_64", + "manylinux_2_17_aarch64.manylinux2014_aarch64", + "macosx_10_13_x86_64", "macosx_11_0_arm64", "win_amd64"): + write_wheel(dist, python=python, platform=platform) + return tmp_path + + +def test_complete_release(bundle): + manifest = release.build_manifest(bundle, PROVENANCE) + assert manifest["version"] == VERSION + assert manifest["source"] == PROVENANCE + assert len(manifest["files"]) == 26 + assert manifest["files"] == sorted(manifest["files"], key=lambda entry: entry["filename"]) + for entry in manifest["files"]: + data = (bundle / "dist" / entry["filename"]).read_bytes() + assert entry["size"] == len(data) + assert entry["sha256"] == hashlib.sha256(data).hexdigest() + + +def test_missing_wheel(bundle): + next((bundle / "dist").glob("*.whl")).unlink() + with pytest.raises(ValueError, match="Incomplete wheel matrix"): + release.build_manifest(bundle, PROVENANCE) + + +def test_duplicate_coverage(bundle): + write_wheel(bundle / "dist", platform="macosx_10_15_x86_64") + with pytest.raises(ValueError, match="Duplicate wheel coverage"): + release.build_manifest(bundle, PROVENANCE) + + +@pytest.mark.parametrize("kwargs, message", [ + ({"version": "6.0"}, "name/version"), + ({"metadata_version": "6.0"}, "metadata mismatch"), + ({"compiled": False}, "no compiled"), + ({"python": "cp314"}, "Unexpected Python ABI"), + ({"abi": "abi3"}, "Unexpected Python ABI"), + ({"python": "py3", "abi": "none", "platform": "any"}, "Unexpected Python ABI"), + ({"platform": "linux_x86_64"}, "Unexpected wheel platform"), + ({"platform": "musllinux_1_2_x86_64"}, "Unexpected wheel platform"), +]) +def test_reject_bad_wheel(tmp_path, kwargs, message): + path = write_wheel(tmp_path, **kwargs) + with pytest.raises(ValueError, match=message): + release.validate_wheel(path, release.Version(VERSION)) + + +@pytest.mark.parametrize("missing", ["pynuodb/_fetch.c", "pynuodb/_cutil.h", + "pynuodb/_resultset.pxi", "tests/conftest.py"]) +def test_missing_sdist_source(tmp_path, missing): + path = write_sdist(tmp_path, missing) + with pytest.raises(ValueError, match="Source distribution is missing"): + release.validate_sdist(path) + + +def test_wrong_tag(bundle): + with pytest.raises(ValueError, match="tag does not match"): + release.build_manifest(bundle, {"ref": "refs/tags/v6.0"}) + + +def test_unexpected_file(bundle): + (bundle / "dist" / "notes.txt").write_text("not a distribution") + with pytest.raises(ValueError, match="only wheel files"): + release.build_manifest(bundle, PROVENANCE) + + +def test_missing_sdist(bundle): + next((bundle / "dist").glob("*.tar.gz")).unlink() + with pytest.raises(ValueError, match="one source archive"): + release.build_manifest(bundle, PROVENANCE) + + +def test_cli_writes_manifest_and_checksums(bundle, monkeypatch): + env = { + "GITHUB_REPOSITORY": "nuodb/nuodb-python", "GITHUB_SHA": "a" * 40, + "GITHUB_REF": "refs/tags/v5.0.0", "GITHUB_EVENT_NAME": "push", + "GITHUB_RUN_ID": "123", "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_WORKFLOW_REF": "nuodb/nuodb-python/.github/workflows/wheels.yml@refs/tags/v5.0.0", + "GITHUB_SERVER_URL": "https://github.com", + } + for name, value in env.items(): + monkeypatch.setenv(name, value) + monkeypatch.setattr("sys.argv", ["release_bundle.py", str(bundle)]) + release.main() + manifest = json.loads((bundle / "manifest.json").read_text()) + assert manifest["source"]["run_attempt"] == "2" + assert manifest["source"]["run_url"] == "https://github.com/nuodb/nuodb-python/actions/runs/123" + checksums = (bundle / "SHA256SUMS").read_text().splitlines() + assert len(checksums) == 26 + for line, entry in zip(checksums, manifest["files"]): + assert line == "{} dist/{}".format(entry["sha256"], entry["filename"]) diff --git a/pyproject.toml b/pyproject.toml index d8aad1e..9966a57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,12 +3,24 @@ requires = ["setuptools>=40.8.0", "wheel", "Cython>=3.0"] build-backend = "setuptools.build_meta" [tool.cibuildwheel] +# Keep the release matrix explicit: new Python ABIs need testing before shipping. +build = ["cp39-*", "cp310-*", "cp311-*", "cp312-*", "cp313-*"] # PyPy: untested; the extension relies on CPython C-API details (e.g. GC # tracking of row tuples) that PyPy's cpyext only emulates. # win32/*_i686: untested 32-bit targets; the MSVC portability shim in # _cutil.h also only implements the 64-bit _BitScanReverse64 intrinsic. -skip = ["pp*", "*-win32", "*-manylinux_i686", "*-musllinux_i686"] +skip = ["pp*", "*-win32", "*-manylinux_i686", "*-musllinux_*"] -# Smoke-test the compiled extension. setup.py builds it with optional=True, -# so a wheel whose compile failed would otherwise silently ship pure Python. -test-command = "python -c \"import pynuodb; import pynuodb._fetch\"" +# Test the installed wheel outside the checkout, not an in-place extension. +# tzdata supplies the IANA database on Windows. No database/server is needed. +test-requires = ["pytest>=8,<9", "tzdata"] +test-sources = ["tests"] +test-command = [ + 'python -c "import pynuodb._fetch"', + 'python -m pytest -v -m "not integration" tests/nuodb_cython_test.py tests/nuodb_crypt_test.py tests/nuodb_datatype_ticks_test.py', +] + +[tool.cibuildwheel.linux] +# glibc >= 2.17; Alpine/musl wheels are not part of the initial release matrix. +manylinux-x86_64-image = "manylinux2014" +manylinux-aarch64-image = "manylinux2014" diff --git a/setup.py b/setup.py index f83e273..e5129b3 100644 --- a/setup.py +++ b/setup.py @@ -108,6 +108,7 @@ def _find_extensions(): url='https://github.com/nuodb/nuodb-python', license='BSD License', long_description=open(readme).read(), + long_description_content_type='text/x-rst', python_requires='>=3.9', install_requires=['tzlocal', 'jdcal', 'cryptography>=36.0'], classifiers=[ diff --git a/tests/conftest.py b/tests/conftest.py index 647de3c..ebfa7f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,6 +52,9 @@ def pytest_addoption(parser): def pytest_configure(config): + config.addinivalue_line( + "markers", + "integration: requires a running NuoDB database") config.addinivalue_line( "markers", "perf: performance benchmark; skipped unless --run-perf is passed") diff --git a/tests/nuodb_cython_test.py b/tests/nuodb_cython_test.py index 2febc21..061b4b4 100644 --- a/tests/nuodb_cython_test.py +++ b/tests/nuodb_cython_test.py @@ -867,6 +867,7 @@ def test_decode_batch_results_non_integer_code_raises(): """ +@pytest.mark.integration class TestNuoDBCython(nuodb_base.NuoBase): def test_cython_matches_pure_python(self): """fetchall() results must be byte-identical between the Cython From c839de10ab88ba70ed98ab8f033d53cebe4d376c Mon Sep 17 00:00:00 2001 From: Eusebio Resende Date: Thu, 22 Oct 2026 17:34:54 +0100 Subject: [PATCH 2/3] Consolidate wheel builds and release bundles on CircleCI --- .circleci/config.yml | 203 ++++++++++++++++++++++++++++++++++- .github/workflows/wheels.yml | 121 --------------------- CONTRIBUTING.rst | 82 ++++++++------ ci/release_bundle.py | 40 +++++-- ci/test_circleci.py | 88 +++++++++++++++ ci/test_release_bundle.py | 62 +++++++++-- test_requirements.txt | 1 + 7 files changed, 423 insertions(+), 174 deletions(-) delete mode 100644 .github/workflows/wheels.yml create mode 100644 ci/test_circleci.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 1cc98ef..92d54e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,7 +24,154 @@ commands: - store_artifacts: path: artifacts + build_wheels: + description: "Build and test wheels from the shared source archive (no PyPI credentials)" + parameters: + target: + type: string + platform: + type: enum + enum: [linux, macos, windows] + python: + type: string + default: .ci-venv/bin/python + steps: + - attach_workspace: + at: workspace + - run: + name: Build and test << parameters.target >> wheels + command: | + << parameters.python >> -m pip install cibuildwheel==4.3.4 + << parameters.python >> -m cibuildwheel --platform << parameters.platform >> \ + --output-dir workspace/wheels/<< parameters.target >> workspace/sdist/*.tar.gz + # Each parallel job persists a distinct directory; never re-persist sdist/. + - persist_to_workspace: + root: workspace + paths: + - wheels/<< parameters.target >> + - store_artifacts: + path: workspace/wheels/<< parameters.target >> + jobs: + build_sdist: + description: "Build and validate the single source archive used by every wheel job" + docker: + - image: cimg/python:3.12 + resource_class: small + steps: + - checkout + - run: + name: Test CI helpers and build the source distribution + command: | + python -m pip install build==1.6.1 twine==7.0.0 packaging 'pytest>=8,<9' 'PyYAML>=6,<7' + python -m pytest -v ci + python -m build --sdist --outdir workspace/sdist + python -m twine check --strict workspace/sdist/* + - run: + name: Check source installation without a compiler + command: | + python -m venv /tmp/fallback-venv + CC=/no-c-compiler /tmp/fallback-venv/bin/python -m pip install --no-cache-dir workspace/sdist/*.tar.gz + cd /tmp + /tmp/fallback-venv/bin/python -c 'import pynuodb; from pynuodb import encodedsession; assert not encodedsession._HAVE_FETCH_ACCEL' + - persist_to_workspace: + root: workspace + paths: [sdist] + - store_artifacts: + path: workspace/sdist + + wheel_linux: + parameters: + arch: + type: enum + enum: [x86_64, aarch64] + resource_class: + type: enum + enum: [medium, arm.medium] + machine: + image: ubuntu-2404:current + resource_class: << parameters.resource_class >> + environment: + CIBW_ARCHS: << parameters.arch >> + steps: + - run: + name: Prepare wheel tooling environment + command: | + sudo apt-get update + sudo apt-get install -y python3-venv + /usr/bin/python3 -m venv .ci-venv + - build_wheels: + target: linux-<< parameters.arch >> + platform: linux + + wheel_macos: + parameters: + arch: + type: enum + enum: [x86_64, arm64] + macos: + xcode: 16.4.0 + resource_class: m4pro.medium + environment: + CIBW_ARCHS: << parameters.arch >> + steps: + - run: + name: Prepare Python and Rosetta for macOS wheel tests + command: | + brew install python@3.12 + /opt/homebrew/opt/python@3.12/bin/python3.12 -m venv .ci-venv + # cibuildwheel runs Intel wheel tests under Rosetta on Apple Silicon. + # Fail here if x86_64 execution is unavailable; do not skip tests. + sudo softwareupdate --install-rosetta --agree-to-license + arch -x86_64 /usr/bin/true + - build_wheels: + target: macos-<< parameters.arch >> + platform: macos + + wheel_windows: + machine: + image: windows-server-2022-gui:current + resource_class: windows.medium + shell: bash.exe --login -eo pipefail + environment: + CIBW_ARCHS: AMD64 + steps: + - run: + name: Prepare wheel tooling environment + command: | + MSYS_NO_PATHCONV=1 choco install python312 --yes --no-progress --params '/InstallDir:C:\Python312' + /c/Python312/python.exe -m venv .ci-venv + - build_wheels: + target: windows-amd64 + platform: windows + python: .ci-venv/Scripts/python.exe + + assemble_release_bundle: + description: "Validate and store a complete bundle only after all builds and tests pass" + docker: + - image: cimg/python:3.12 + resource_class: small + environment: + RELEASE_PIPELINE_ID: << pipeline.id >> + RELEASE_PIPELINE_NUMBER: << pipeline.number >> + steps: + - checkout + - attach_workspace: + at: workspace + - run: + name: Validate distributions and record CircleCI provenance + command: | + mkdir -p release-bundle/dist + cp workspace/sdist/*.tar.gz workspace/wheels/*/*.whl release-bundle/dist/ + python -m pip install twine==7.0.0 packaging + python -m twine check --strict release-bundle/dist/* + python ci/release_bundle.py release-bundle + (cd release-bundle && sha256sum --check SHA256SUMS) + tar -czf release-bundle.tar.gz -C release-bundle . + - store_artifacts: + path: release-bundle.tar.gz + destination: release-bundle.tar.gz + build_n_run: description: "Build the nuodb-python module and run the test suite" docker: @@ -54,12 +201,16 @@ jobs: - run: name: Make test directories command: mkdir -p artifacts results + - attach_workspace: + at: workspace - run: name: Build, validate, and install the regression wheel command: | - python3 -m pip install 'build>=1.2,<2' 'twine>=6,<8' -r test_requirements.txt - # The default build creates an sdist, then builds the wheel from it. - python3 -m build --outdir artifacts/dist + python3 -m pip install 'twine>=6,<8' -r test_requirements.txt + mkdir -p artifacts/dist + cp workspace/sdist/*.tar.gz artifacts/dist/ + # Exercise the same source archive used for every release wheel. + python3 -m pip wheel --no-deps --wheel-dir artifacts/dist workspace/sdist/*.tar.gz python3 -m twine check --strict artifacts/dist/* python3 -m pip install --force-reinstall artifacts/dist/*.whl - run: @@ -167,11 +318,57 @@ jobs: workflows: build-project: jobs: + # Enable version tags on every upstream job, not just bundle assembly. + - build_sdist: + name: "Build source distribution" + filters: &build-filters + tags: + only: /^v.*/ + - wheel_linux: + name: "Wheels (linux-x86_64)" + arch: x86_64 + resource_class: medium + requires: ["Build source distribution"] + filters: *build-filters + - wheel_linux: + name: "Wheels (linux-aarch64)" + arch: aarch64 + resource_class: arm.medium + requires: ["Build source distribution"] + filters: *build-filters + - wheel_macos: + name: "Wheels (macos-x86_64)" + arch: x86_64 + requires: ["Build source distribution"] + filters: *build-filters + - wheel_macos: + name: "Wheels (macos-arm64)" + arch: arm64 + requires: ["Build source distribution"] + filters: *build-filters + - wheel_windows: + name: "Wheels (windows-amd64)" + requires: ["Build source distribution"] + filters: *build-filters - build_n_run: name: "Build and run regression tests" + requires: ["Build source distribution"] + filters: *build-filters context: - common-config - perf_bench: name: "Run performance benchmarks, comparing results to master" + filters: *build-filters context: - common-config + - assemble_release_bundle: + name: "Validate and assemble release bundle" + requires: + - "Wheels (linux-x86_64)" + - "Wheels (linux-aarch64)" + - "Wheels (macos-x86_64)" + - "Wheels (macos-arm64)" + - "Wheels (windows-amd64)" + - "Build and run regression tests" + - "Run performance benchmarks, comparing results to master" + filters: *build-filters diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml deleted file mode 100644 index 663b898..0000000 --- a/.github/workflows/wheels.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: Build distribution artifacts - -on: - pull_request: - push: - branches: [master] - tags: ['v*'] - workflow_dispatch: - -# No publishing credentials or OIDC permissions belong in this workflow. -permissions: - contents: read - -jobs: - sdist: - name: Build source distribution - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: '3.11' - - name: Install release tooling - run: python -m pip install build==1.6.1 twine==7.0.0 packaging 'pytest>=8,<9' - - name: Test release-bundle validation - run: python -m pytest -v ci/test_release_bundle.py - - name: Build and check source distribution - run: | - python -m build --sdist - python -m twine check --strict dist/* - - name: Check source installation without a compiler - shell: bash - run: | - CC=/no-c-compiler python -m pip install --no-cache-dir dist/*.tar.gz - cd "$RUNNER_TEMP" - python -c 'import pynuodb; from pynuodb import encodedsession; assert not encodedsession._HAVE_FETCH_ACCEL' - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: sdist - path: dist/*.tar.gz - if-no-files-found: error - retention-days: 90 - - wheels: - name: Wheels (${{ matrix.target }}) - needs: sdist - strategy: - fail-fast: false - matrix: - include: - - target: linux-x86_64 - os: ubuntu-24.04 - arch: x86_64 - - target: linux-aarch64 - os: ubuntu-24.04-arm - arch: aarch64 - - target: macos-x86_64 - os: macos-15-intel - arch: x86_64 - - target: macos-arm64 - os: macos-15 - arch: arm64 - - target: windows-amd64 - os: windows-2022 - arch: AMD64 - runs-on: ${{ matrix.os }} - steps: - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: '3.11' - - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 - with: - name: sdist - path: dist - - name: Install wheel tooling - run: python -m pip install cibuildwheel==4.3.4 - - name: Build and test wheels from the source distribution - shell: bash - env: - CIBW_ARCHS: ${{ matrix.arch }} - run: python -m cibuildwheel --output-dir wheelhouse dist/*.tar.gz - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: wheels-${{ matrix.target }} - path: wheelhouse/*.whl - if-no-files-found: error - retention-days: 90 - - release-bundle: - name: Validate and assemble release bundle - needs: [sdist, wheels] - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: '3.11' - - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 - with: - name: sdist - path: release-bundle/dist - - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 - with: - pattern: wheels-* - merge-multiple: true - path: release-bundle/dist - - name: Validate distributions and record provenance - run: | - python -m pip install twine==7.0.0 packaging - python -m twine check --strict release-bundle/dist/* - python ci/release_bundle.py release-bundle - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: release-bundle-${{ github.run_id }}-${{ github.run_attempt }} - path: release-bundle/ - if-no-files-found: error - retention-days: 90 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6c9cdfb..e2b8775 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -81,17 +81,20 @@ Maintain the list of changes per release in CHANGES.rst. Also note the known def Building distribution artifacts ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``.github/workflows/wheels.yml`` builds distribution artifacts on pull requests, -pushes to ``master``, version tags matching ``v*``, and manual workflow runs. -It does **not** publish to PyPI, request an OIDC token, or use PyPI credentials. -The internal deployment process remains responsible for authorizing a release -and publishing it. +``.circleci/config.yml`` owns the complete build, test, and artifact workflow. +It runs on branch builds and version tags matching ``v*``, subject to the +project's CircleCI trigger settings, and can be triggered manually in CircleCI. +It does **not** publish to PyPI or use PyPI credentials. The internal deployment +process remains responsible for authorizing a release and publishing it. +There is no separate GitHub Actions build workflow. The initial wheel matrix is CPython 3.9 through 3.13 (standard, GIL-enabled builds) on: * Linux x86-64 and ARM64, using ``manylinux2014`` (glibc >= 2.17). * macOS Intel and Apple Silicon, using separate architecture-specific wheels. + Both jobs use Apple Silicon runners; Intel wheel tests execute under Rosetta. + Rosetta installation and x86-64 execution are checked explicitly, not skipped. * Windows x86-64. PyPy, free-threaded Python, 32-bit targets, Windows ARM64, and Alpine/musl @@ -108,17 +111,27 @@ The tests import the installed wheel, not the source checkout, and fail if the compiled extension is missing. Windows tests install ``tzdata`` for IANA time zones. -CircleCI continues to run the full database regression suite and performance -comparison. Its regression job now builds and installs a wheel from an sdist -and runs the tests outside the checkout. The job saves its native wheel and -source archive under ``artifacts/dist`` for diagnosis; that native Linux wheel -is **not** a portable manylinux release artifact. +The database regression job builds and installs a wheel from that same source +archive and runs the full suite outside the checkout. It saves its native wheel +and source archive under ``artifacts/dist`` for diagnosis; that native Linux +wheel is **not** a portable manylinux release artifact. The existing performance +comparison also runs in the same workflow. + +Linux wheel jobs use x86-64 and ARM64 machine executors with Docker for +manylinux builds. macOS jobs use ``m4pro.medium`` and Windows uses +``windows.medium``. These resource classes must be enabled for the CircleCI +organization and consume the corresponding credits. Only the existing database +and performance jobs use the ``common-config`` context; source/wheel building +and bundle assembly do not use a secret context. Release bundle ^^^^^^^^^^^^^^ -Only after all GitHub wheel jobs pass does the workflow assemble the artifact -``release-bundle--``. It contains:: +Only after all five wheel jobs, database regressions, and performance tests +pass does ``Validate and assemble release bundle`` run. CircleCI workspaces +carry the source archive and each target's wheels between jobs, using distinct +paths for parallel outputs. The final job stores one downloadable artifact, +``release-bundle.tar.gz``. Extracting it produces:: dist/ pynuodb-.tar.gz @@ -135,22 +148,25 @@ contents. On tag runs, the tag version must match the package version distribution metadata. The manifest records distribution sizes and SHA-256 checksums, the source -commit/ref, repository, workflow ref, event, run ID, and run attempt. Artifacts -request 90-day retention (subject to repository/organization limits); archive -approved bundles internally before they expire. Dependency resolution and -builder images can change, so a rebuild is not guaranteed to be byte-identical. +commit/ref, repository, CircleCI pipeline ID/number, workflow ID, and the exact +artifact-producing job number and URL. A rerun produces a new job identity; +do not select artifacts by version or branch name alone. A pull-request URL is +recorded when CircleCI supplies one, but its absence is not release approval. +Artifacts are subject to CircleCI retention settings; archive approved bundles +internally before they expire. Dependency resolution and builder images can +change, so a rebuild is not guaranteed to be byte-identical. For internal publishing: -#. Select a successful run of this workflow for an approved release commit or - protected version tag. Use its exact run ID and attempt, not "latest". -#. Confirm that CircleCI database regressions passed for the same release - commit. GitHub's bundle job does not automatically gate on CircleCI status. -#. Download the **final release bundle**, not the intermediate wheel artifacts - or CircleCI's diagnostic distributions. Check its manifest against the - approved repository, commit, version, and run identity. Pull-request runs - build a merge commit for testing and must not be used for publication. -#. Verify the downloaded distributions from the bundle directory:: +#. Select a successful CircleCI workflow for an approved release commit or + protected version tag. Use its exact workflow ID and final job number, + not "latest". Bundle assembly is automatically gated on builds and tests. +#. Download ``release-bundle.tar.gz`` from that workflow's final assembly job, + not the intermediate wheel artifacts or diagnostic distributions. Extract + it and check the manifest against the approved repository, commit, version, + pipeline, workflow, and job identity. Branch/PR test artifacts must not be + treated as approved releases merely because their tests passed. +#. Verify the downloaded distributions from the extracted bundle directory:: sha256sum --check SHA256SUMS @@ -158,13 +174,15 @@ For internal publishing: separate publishing job. Do not rebuild distributions during publishing. Checksums detect corruption; they do not authenticate an untrusted build or -replace release approval. No GitHub-to-internal-GitLab connection is required: -the internal runner can fetch the selected artifact using outbound access. +replace release approval. No inbound connection to internal GitLab is required: +the internal runner can fetch the selected CircleCI artifact using outbound +access (with a read-only CircleCI credential if required for artifact access). The PyPI token must never be supplied to this workflow or the wheel tests. -When adding Python versions or targets, update ``pyproject.toml``, the workflow -matrix, and ``PYTHONS``/``TARGETS`` in ``ci/release_bundle.py`` together. Test the -bundle validator locally with:: +When adding Python versions or targets, update ``pyproject.toml``, the CircleCI +workflow jobs, and ``PYTHONS``/``TARGETS`` in ``ci/release_bundle.py`` together. +Test the bundle validator and workflow invariants locally with:: - python -m pip install packaging 'pytest>=8,<9' - python -m pytest ci/test_release_bundle.py + python -m pip install packaging 'pytest>=8,<9' 'PyYAML>=6,<7' + python -m pytest ci + circleci config validate .circleci/config.yml diff --git a/ci/release_bundle.py b/ci/release_bundle.py index eaa5a87..233d94d 100644 --- a/ci/release_bundle.py +++ b/ci/release_bundle.py @@ -17,7 +17,7 @@ from packaging.version import Version -# Keep in sync with pyproject.toml and .github/workflows/wheels.yml. +# Keep in sync with pyproject.toml and .circleci/config.yml. PYTHONS = ("cp39", "cp310", "cp311", "cp312", "cp313") TARGETS = ("linux-x86_64", "linux-aarch64", "macos-x86_64", "macos-arm64", "windows-amd64") @@ -138,18 +138,40 @@ def build_manifest(bundle, provenance): } +def circleci_provenance(): + """Record the exact workflow and artifact-producing job, including reruns.""" + required = ( + "CIRCLE_PROJECT_USERNAME", "CIRCLE_PROJECT_REPONAME", "CIRCLE_SHA1", + "CIRCLE_WORKFLOW_ID", "CIRCLE_BUILD_NUM", "CIRCLE_BUILD_URL", "CIRCLE_JOB", + "RELEASE_PIPELINE_ID", "RELEASE_PIPELINE_NUMBER", + ) + missing = [name for name in required if not os.environ.get(name)] + if missing: + raise ValueError("Missing CircleCI provenance: " + ", ".join(missing)) + tag, branch = os.environ.get("CIRCLE_TAG"), os.environ.get("CIRCLE_BRANCH") + if not tag and not branch: + raise ValueError("CircleCI must identify a tag or branch") + return { + "provider": "circleci", + "repository": os.environ["CIRCLE_PROJECT_USERNAME"] + "/" + os.environ["CIRCLE_PROJECT_REPONAME"], + "commit": os.environ["CIRCLE_SHA1"], + "ref": "refs/tags/" + tag if tag else "refs/heads/" + branch, + "pipeline_id": os.environ["RELEASE_PIPELINE_ID"], + "pipeline_number": os.environ["RELEASE_PIPELINE_NUMBER"], + "workflow_id": os.environ["CIRCLE_WORKFLOW_ID"], + "job_number": os.environ["CIRCLE_BUILD_NUM"], + "job_name": os.environ["CIRCLE_JOB"], + "job_url": os.environ["CIRCLE_BUILD_URL"], + # Informational only: an empty PR URL is NOT authorization to publish. + "pull_request": os.environ.get("CIRCLE_PULL_REQUEST", ""), + } + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("bundle", type=Path, help="directory containing dist/") args = parser.parse_args() - provenance = {key: os.environ[env] for key, env in { - "repository": "GITHUB_REPOSITORY", "commit": "GITHUB_SHA", "ref": "GITHUB_REF", - "event": "GITHUB_EVENT_NAME", "run_id": "GITHUB_RUN_ID", - "run_attempt": "GITHUB_RUN_ATTEMPT", "workflow_ref": "GITHUB_WORKFLOW_REF", - }.items()} - provenance["run_url"] = "{}/{}/actions/runs/{}".format( - os.environ["GITHUB_SERVER_URL"], provenance["repository"], provenance["run_id"]) - manifest = build_manifest(args.bundle, provenance) + manifest = build_manifest(args.bundle, circleci_provenance()) (args.bundle / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") checksums = "".join("{} dist/{}\n".format(file["sha256"], file["filename"]) for file in manifest["files"]) diff --git a/ci/test_circleci.py b/ci/test_circleci.py new file mode 100644 index 0000000..47ffa26 --- /dev/null +++ b/ci/test_circleci.py @@ -0,0 +1,88 @@ +"""Guard the release matrix and the fan-in gate against configuration drift.""" + +from pathlib import Path + +import pytest +import yaml + +from ci import release_bundle as release + + +@pytest.fixture +def config(): + root = Path(__file__).resolve().parents[1] + return yaml.safe_load((root / ".circleci/config.yml").read_text()) + + +def workflow_jobs(config): + return [next(iter(job.items())) for job in config["workflows"]["build-project"]["jobs"]] + + +def test_wheel_targets_match_validator(config): + targets = set() + for kind, job in workflow_jobs(config): + if kind in ("wheel_linux", "wheel_macos"): + platform = kind.removeprefix("wheel_") + targets.add(platform + "-" + job["arch"]) + elif kind == "wheel_windows": + targets.add("windows-amd64") + assert targets == set(release.TARGETS) + + +def test_bundle_requires_all_wheels_and_tests(config): + jobs = workflow_jobs(config) + bundle = next(job for kind, job in jobs if kind == "assemble_release_bundle") + required = {job["name"] for kind, job in jobs + if kind.startswith("wheel_") or kind in ("build_n_run", "perf_bench")} + assert set(bundle["requires"]) == required + + +def test_all_jobs_enable_version_tags(config): + for _, job in workflow_jobs(config): + assert job["filters"]["tags"]["only"] == "/^v.*/" + + +def test_builds_and_regressions_use_shared_sdist(config): + source = next(job["name"] for kind, job in workflow_jobs(config) if kind == "build_sdist") + for kind, job in workflow_jobs(config): + if kind.startswith("wheel_") or kind == "build_n_run": + assert source in job["requires"] + command = config["commands"]["build_wheels"]["steps"][1]["run"]["command"] + assert "workspace/sdist/*.tar.gz" in command + assert "--platform << parameters.platform >>" in command + regression = config["jobs"]["build_n_run"]["steps"] + build = next(step["run"]["command"] for step in regression + if isinstance(step, dict) and "run" in step + and step["run"]["name"] == "Build, validate, and install the regression wheel") + assert "pip wheel --no-deps" in build + assert "workspace/sdist/*.tar.gz" in build + + +def test_wheel_architectures_and_runners_match(config): + for kind, job in workflow_jobs(config): + if kind == "wheel_linux": + assert job["resource_class"] == {"x86_64": "medium", "aarch64": "arm.medium"}[job["arch"]] + assert config["jobs"]["wheel_macos"]["resource_class"].startswith("m4pro.") + assert config["jobs"]["wheel_windows"]["environment"]["CIBW_ARCHS"] == "AMD64" + assert config["jobs"]["wheel_windows"]["shell"].startswith("bash.exe") + + +def test_wheel_tests_are_not_skipped(config): + for kind in ("wheel_linux", "wheel_macos", "wheel_windows"): + assert not any(name.startswith("CIBW_TEST_SKIP") + for name in config["jobs"][kind].get("environment", {})) + setup = config["jobs"]["wheel_macos"]["steps"][0]["run"]["command"] + assert "--install-rosetta" in setup + assert "arch -x86_64 /usr/bin/true" in setup + + +def test_no_secret_context_on_release_build_jobs(config): + for kind, job in workflow_jobs(config): + if kind not in ("build_n_run", "perf_bench"): + assert "context" not in job + + +def test_parallel_workspace_paths_are_distinct(config): + steps = config["commands"]["build_wheels"]["steps"] + persisted = next(step["persist_to_workspace"] for step in steps if "persist_to_workspace" in step) + assert persisted == {"root": "workspace", "paths": ["wheels/<< parameters.target >>"]} diff --git a/ci/test_release_bundle.py b/ci/test_release_bundle.py index e10e3b4..60de124 100644 --- a/ci/test_release_bundle.py +++ b/ci/test_release_bundle.py @@ -12,7 +12,7 @@ VERSION = "5.0" -PROVENANCE = {"ref": "refs/tags/v5.0", "commit": "a" * 40, "run_id": "123"} +PROVENANCE = {"ref": "refs/tags/v5.0", "commit": "a" * 40, "workflow_id": "workflow-123"} def metadata(version=VERSION): @@ -122,21 +122,65 @@ def test_missing_sdist(bundle): release.build_manifest(bundle, PROVENANCE) -def test_cli_writes_manifest_and_checksums(bundle, monkeypatch): +@pytest.fixture +def circleci_env(monkeypatch): env = { - "GITHUB_REPOSITORY": "nuodb/nuodb-python", "GITHUB_SHA": "a" * 40, - "GITHUB_REF": "refs/tags/v5.0.0", "GITHUB_EVENT_NAME": "push", - "GITHUB_RUN_ID": "123", "GITHUB_RUN_ATTEMPT": "2", - "GITHUB_WORKFLOW_REF": "nuodb/nuodb-python/.github/workflows/wheels.yml@refs/tags/v5.0.0", - "GITHUB_SERVER_URL": "https://github.com", + "CIRCLE_PROJECT_USERNAME": "nuodb", "CIRCLE_PROJECT_REPONAME": "nuodb-python", + "CIRCLE_SHA1": "a" * 40, "CIRCLE_TAG": "v5.0.0", "CIRCLE_BRANCH": "", + "CIRCLE_WORKFLOW_ID": "workflow-123", "CIRCLE_BUILD_NUM": "456", + "CIRCLE_BUILD_URL": "https://circleci.com/gh/nuodb/nuodb-python/456", + "CIRCLE_JOB": "assemble_release_bundle", "CIRCLE_PULL_REQUEST": "", + "RELEASE_PIPELINE_ID": "pipeline-789", "RELEASE_PIPELINE_NUMBER": "789", } for name, value in env.items(): monkeypatch.setenv(name, value) + return env + + +def test_circleci_provenance(circleci_env): + source = release.circleci_provenance() + assert source["provider"] == "circleci" + assert source["repository"] == "nuodb/nuodb-python" + assert source["ref"] == "refs/tags/v5.0.0" + assert source["commit"] == "a" * 40 + assert source["pipeline_id"] == "pipeline-789" + assert source["workflow_id"] == "workflow-123" + + +def test_branch_provenance(circleci_env, monkeypatch): + monkeypatch.setenv("CIRCLE_TAG", "") + monkeypatch.setenv("CIRCLE_BRANCH", "ci/platform-wheel-artifacts") + monkeypatch.setenv("CIRCLE_PULL_REQUEST", "https://github.com/nuodb/nuodb-python/pull/218") + source = release.circleci_provenance() + assert source["ref"] == "refs/heads/ci/platform-wheel-artifacts" + assert source["pull_request"].endswith("/218") + + +def test_tag_takes_precedence(circleci_env, monkeypatch): + monkeypatch.setenv("CIRCLE_BRANCH", "master") + assert release.circleci_provenance()["ref"] == "refs/tags/v5.0.0" + + +@pytest.mark.parametrize("missing", ["CIRCLE_SHA1", "CIRCLE_WORKFLOW_ID", "CIRCLE_BUILD_NUM", + "RELEASE_PIPELINE_ID", "RELEASE_PIPELINE_NUMBER"]) +def test_missing_provenance(circleci_env, monkeypatch, missing): + monkeypatch.delenv(missing) + with pytest.raises(ValueError, match="Missing CircleCI provenance"): + release.circleci_provenance() + + +def test_missing_ref(circleci_env, monkeypatch): + monkeypatch.setenv("CIRCLE_TAG", "") + with pytest.raises(ValueError, match="identify a tag or branch"): + release.circleci_provenance() + + +def test_cli_writes_manifest_and_checksums(bundle, monkeypatch, circleci_env): monkeypatch.setattr("sys.argv", ["release_bundle.py", str(bundle)]) release.main() manifest = json.loads((bundle / "manifest.json").read_text()) - assert manifest["source"]["run_attempt"] == "2" - assert manifest["source"]["run_url"] == "https://github.com/nuodb/nuodb-python/actions/runs/123" + assert manifest["source"]["job_number"] == "456" + assert manifest["source"]["job_url"] == "https://circleci.com/gh/nuodb/nuodb-python/456" checksums = (bundle / "SHA256SUMS").read_text().splitlines() assert len(checksums) == 26 for line, entry in zip(checksums, manifest["files"]): diff --git a/test_requirements.txt b/test_requirements.txt index 2328d32..9ae8ab9 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,6 +1,7 @@ mock>=1.0 nose>=1.3 pytest>=2.7 +PyYAML>=6,<7 pytest-benchmark>=4.0 coverage>=3.7 pytest-cov>=1.8.1 From 47839da09cfa4eea272099bd3b4be2ac0d77ebba Mon Sep 17 00:00:00 2001 From: Eusebio Resende Date: Thu, 22 Oct 2026 18:15:04 +0100 Subject: [PATCH 3/3] Document wheel installation, test coverage, and filename tags --- CONTRIBUTING.rst | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ README.rst | 44 ++++++++++++++++++++++++++++++++--------- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e2b8775..3c3184b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -124,6 +124,57 @@ organization and consume the corresponding credits. Only the existing database and performance jobs use the ``common-config`` context; source/wheel building and bundle assembly do not use a secret context. +Understanding wheel filenames +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A wheel is a pre-built, installable Python distribution. For this driver it +contains both Python modules and the compiled accelerator, so users installing +a compatible wheel do not need to compile the extension themselves. + +The current driver wheels follow this filename pattern:: + + pynuodb----.whl + +For example:: + + pynuodb-5.0.0-cp39-cp39-win_amd64.whl + +.. list-table:: Reading the example filename + :header-rows: 1 + :widths: 30 70 + + * - Component + - Meaning + * - ``pynuodb`` + - Package name. + * - ``5.0.0`` + - Driver version. + * - First ``cp39`` + - Python interpreter: CPython 3.9. + * - Second ``cp39`` + - Application binary interface (ABI): the CPython 3.9 interface that + the compiled extension must match. + * - ``win_amd64`` + - Windows on x86-64 CPUs, including both Intel and AMD processors. + +The Python tags ``cp39``, ``cp310``, ``cp311``, ``cp312``, and ``cp313`` identify +CPython 3.9, 3.10, 3.11, 3.12, and 3.13 respectively. Cython is the build-time +tool that translates the extension sources into C; its version is not encoded +in a wheel filename. + +The current extension uses a CPython-version-specific ABI, rather than the +cross-version ``abi3`` tag. Its CPython 3.9 wheel cannot be reused for CPython +3.13, and a Windows x86-64 binary cannot be installed on Linux ARM64. Therefore, +the five Python versions and five OS/CPU targets listed above require +**5 x 5 = 25 wheels**, plus one source archive. Linux uses ``manylinux2014`` +compatibility tags rather than a separate wheel for each compatible glibc-based +Linux distribution. + +All release wheels share the same package name and driver version on PyPI. +Users still run ``python -m pip install pynuodb``; ``pip`` selects a compatible +wheel automatically when one has been published. CI artifacts are build +candidates, not evidence that a version has already been published. + Release bundle ^^^^^^^^^^^^^^ diff --git a/README.rst b/README.rst index baab8cf..8e957b4 100644 --- a/README.rst +++ b/README.rst @@ -7,16 +7,24 @@ NuoDB - Python .. contents:: -This package contains the community driven pure-Python NuoDB_ client library -that provides a standard `PEP 249`_ SQL API. This is a community driven driver -with limited support and testing from NuoDB. +This package contains the community driven NuoDB_ client library that provides +a standard `PEP 249`_ SQL API. It includes a pure-Python implementation and an +optional Cython-based accelerator for encoding parameters and decoding results. +The same Python API is used with or without the accelerator. This is a +community driven driver with limited support and testing from NuoDB. Requirements ------------ * Python >= 3.9 - - Tested with CPython_ 3.9 + - The CI wheel matrix builds and tests standard (GIL-enabled) CPython_ + 3.9, 3.10, 3.11, 3.12, and 3.13 on Linux x86-64/ARM64, macOS Intel/Apple + Silicon, and Windows x86-64. + - Each wheel runs database-independent accelerator, encoding, and date/time + tests. Full database regressions and performance tests run separately on + Linux using the Python version in the NuoDB CI image; they do not run for + every wheel combination. * NuoDB_ >= 6.0.2 @@ -30,15 +38,32 @@ Installation The current stable release is available on PyPI and can be installed with ``pip``:: - $ pip install pynuodb + $ python -m pip install pynuodb -Alternatively (e.g. if ``pip`` is not available), a tarball can be downloaded -from GitHub and installed with Setuptools:: +For a release with compatible wheels on PyPI, ``pip`` automatically selects +the wheel matching your Python version, operating system, and CPU architecture. +There is no separate package name or manual binary selection. Installing a +matching wheel requires neither Cython nor a C compiler; the accelerator is +already compiled into the wheel. + +If no compatible wheel is available and the release provides a source archive, +``pip`` attempts a local build. A C compiler is needed to build the optional +accelerator. If extension compilation is unavailable or fails, installation +can continue with the pure-Python implementation, without acceleration. This +fallback applies to the driver's optional extension: build requirements and +runtime dependencies, including ``cryptography``, must still be installable. + +The CI workflow in this repository generates wheel candidates and a source +archive. Publishing them to PyPI is a separate release step; a successful CI +build does not imply that its wheels are already available on PyPI. See the +`Contributor guide`_ for the build matrix, wheel filenames, and release process. + +To install a development snapshot from GitHub instead of a released version:: $ curl -L https://github.com/nuodb/nuodb-python/archive/master.tar.gz | tar xz $ cd nuodb-python* - $ python setup.py install - # The folder nuodb-python* can be safely removed now. + $ python -m pip install . + # The extracted source folder can be removed after installation. Example ------- @@ -105,3 +130,4 @@ PyNuoDB is licensed under a `BSD 3-Clause License`_. .. _PEP 249: https://www.python.org/dev/peps/pep-0249/ .. _Quick Start Guides / Docker: https://doc.nuodb.com/nuodb/latest/quick-start-guide/docker/ .. _BSD 3-Clause License: https://github.com/nuodb/nuodb-python/blob/master/LICENSE +.. _Contributor guide: https://github.com/nuodb/nuodb-python/blob/master/CONTRIBUTING.rst