From 011810a2c61fc3c4f9723382a4e9103e9d782e83 Mon Sep 17 00:00:00 2001 From: jasper blues Date: Sun, 8 Nov 2026 14:19:29 +1000 Subject: [PATCH 1/5] Split CI's tests over parallel jobs, and move the slow suites to a nightly A pull request's run leaves out the tests marked nightly (timings, Oracle's plans, texts past 32 KB), and runs the rest in parts with pytest --shard. The full set runs on a push to main, in nightly.yml, in the release gate, and on a pull request labelled full-ci. A release trusts only a full run. --- .github/workflows/ci.yml | 117 ++++-- .github/workflows/nightly.yml | 97 +++++ .github/workflows/prepare-release.yml | 3 +- .github/workflows/release.yml | 15 +- RELEASING.md | 8 +- docs/11-testing.md | 8 +- pyproject.toml | 1 + scripts/ci_trust.py | 9 + tests/conftest.py | 7 +- tests/durations.json | 332 ++++++++++++++++++ .../oracle/test_oracle_adaptive_plans.py | 2 +- .../oracle/test_oracle_era_parse.py | 2 +- .../oracle/test_oracle_keyed_label_plan.py | 2 +- .../oracle/test_oracle_long_text_memory.py | 3 +- .../oracle/test_oracle_node_equality.py | 2 +- .../test_compiled_write_keyed_read_time.py | 2 +- .../sql/test_compiled_write_loop_time.py | 2 + tests/integration/test_cypher_compiled.py | 13 +- .../integration/test_object_manager_saving.py | 1 + tests/shards.py | 104 ++++++ .../oracle/test_oracle_adaptive_plans.py | 2 +- .../oracle/test_oracle_era_parse.py | 2 +- .../oracle/test_oracle_keyed_label_plan.py | 2 +- .../oracle/test_oracle_long_text_memory.py | 3 +- .../oracle/test_oracle_node_equality.py | 2 +- .../test_compiled_write_keyed_read_time.py | 2 +- .../sql/test_compiled_write_loop_time.py | 2 + .../sync_integration/test_cypher_compiled.py | 13 +- .../test_object_manager_saving.py | 1 + tests/unit/test_ci_trust.py | 17 + tests/unit/test_shards.py | 37 ++ 31 files changed, 753 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/nightly.yml create mode 100644 tests/durations.json create mode 100644 tests/shards.py create mode 100644 tests/unit/test_shards.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69a34455..5751e658 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,31 +1,46 @@ name: CI +# Lints and tests drivine on each Python it supports, and again with Oracle, each run split over parallel jobs with +# containers of their own: `pytest --shard i/n` runs one part, a module whole in one part (tests/shards.py). +# +# A pull request's run leaves out the nightly tests (`-m nightly`, the tests marked so): slow ones, and ones that rest +# on a timing or a plan. A full run has them too, in a part of their own: a push to main, a pull request labelled +# `full-ci`, nightly.yml each night, and release.yml's gate. release.yml calls this as the gate before a release, +# unless a full push or pull_request run of it has already passed on the same tree. on: push: branches: [main] pull_request: - # release.yml runs this as its gate, so that a release passes the same CI as a pull request, unless a push or - # pull_request run of it has already passed on the same tree. + # As the default, and labeled, so that labelling a pull request `full-ci` runs the full set on it. + types: [opened, synchronize, reopened, labeled] workflow_call: - # Each night on the latest images, without the pin below, so an engine's new release shows up here and not in a pull - # request's run. - schedule: - - cron: "17 4 * * *" + inputs: + full: + description: Run the nightly tests too, as a release and the nightly do. + type: boolean + default: false -# FalkorDB is held at a release that passes: a pull request's run takes falkordb/falkordb:6.0.3. The nightly run takes -# ghcr.io/falkordb/falkordb:latest, FalkorDB's newest release, and not the falkordb/falkordb:latest the plugin names: -# Docker Hub's latest is whichever release FalkorDB pushed last, and that can be an older line's backport, as v4.22.3 -# took it after 6.0.3 (no =~, and no pattern comprehension in a function, FalkorDB/FalkorDB#3551). Drop the nightly's -# ghcr.io image once Docker Hub's latest follows FalkorDB's newest line again; move the pin, or drop it, once the -# nightly passes. +# A pull request's newer run, of a later push or a label, takes the place of the one still going. Any other run, and +# this workflow called by another, is a group of its own. +concurrency: + group: ${{ github.workflow == 'CI' && github.event_name == 'pull_request' && format('ci-pr-{0}', github.event.pull_request.number) || format('ci-{0}-{1}', github.workflow, github.run_id) }} + cancel-in-progress: true + +# FalkorDB is held at a release that passes: a pull request's run takes falkordb/falkordb:6.0.3. The nightly run +# (nightly.yml, whose name this sees as its workflow) takes ghcr.io/falkordb/falkordb:latest, FalkorDB's newest release, +# and not the falkordb/falkordb:latest the plugin names: Docker Hub's latest is whichever release FalkorDB pushed last, +# and that can be an older line's backport, as v4.22.3 took it after 6.0.3 (no =~, and no pattern comprehension in a +# function, FalkorDB/FalkorDB#3551). Drop the nightly's ghcr.io image once Docker Hub's latest follows FalkorDB's newest +# line again; move the pin, or drop it, once the nightly passes. env: - FALKORDB_TEST_IMAGE: ${{ github.event_name == 'schedule' && 'ghcr.io/falkordb/falkordb:latest' || 'falkordb/falkordb:6.0.3' }} + FALKORDB_TEST_IMAGE: ${{ github.workflow == 'Nightly' && 'ghcr.io/falkordb/falkordb:latest' || 'falkordb/falkordb:6.0.3' }} jobs: # Records the git tree this run tests, as an artifact named ci-tree-, so a release of the same tree can trust # this run and skip its gate (scripts/ci_trust.py). It matters on a pull request, which tests its merge ref, not the - # head commit the run names. It has no condition, as no job here may: release.yml trusts a run only when every job - # in it passed, and a job skipped by a condition doesn't. + # head commit the run names. A run of the pull request set alone, without the nightly tests, names it + # ci-tree-pr-set-, which a release doesn't trust. It has no condition, as no job here may: release.yml trusts + # a run only when every job in it passed, and a job skipped by a condition doesn't. tree: name: record the tree runs-on: ubuntu-latest @@ -41,7 +56,7 @@ jobs: # Overwritten when the run is re-run, as an artifact's name is unique within a run. - uses: actions/upload-artifact@v4 with: - name: ci-tree-${{ steps.tree.outputs.tree }} + name: ci-tree-${{ !(inputs.full || github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'full-ci')) && 'pr-set-' || '' }}${{ steps.tree.outputs.tree }} path: ci-tree.txt if-no-files-found: error overwrite: true @@ -58,30 +73,57 @@ jobs: - name: Check the Cypher spec run: python scripts/check_cypher_spec.py + # Every test but Oracle's, on each Python, in three parts, and the nightly tests in a fourth on a full run (picked by + # the same test as `tree`'s record). The first part lints too. The parts of a pull request's run hold every test but + # the nightly ones once. test: + name: test (${{ matrix.python }}, ${{ matrix.part }}) runs-on: ubuntu-latest + timeout-minutes: 30 strategy: fail-fast: false matrix: python: ["3.11", "3.12", "3.13", "3.14"] + part: ${{ fromJSON((inputs.full || github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'full-ci')) && '["1/3", "2/3", "3/3", "nightly"]' || '["1/3", "2/3", "3/3"]') }} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 with: python-version: ${{ matrix.python }} - run: uv sync - - run: uv run ruff check . - - run: uv run ruff format --check . - - run: uv run pyright - - run: uv run python scripts/unasync.py --check - - run: uv run pytest -q + - if: matrix.part == '1/3' + run: uv run ruff check . + - if: matrix.part == '1/3' + run: uv run ruff format --check . + - if: matrix.part == '1/3' + run: uv run pyright + - if: matrix.part == '1/3' + run: uv run python scripts/unasync.py --check + - name: Run the tests + env: + PART: ${{ matrix.part }} + run: | + if [ "$PART" = nightly ]; then picked=(-m nightly); else picked=(-m "not nightly" --shard "$PART"); fi + uv run pytest -q --durations=15 --record-durations durations.json "${picked[@]}" + # How long each module took, to balance the parts by (tests/durations.json). From the slowest Python alone. + - if: always() && matrix.python == '3.11' + uses: actions/upload-artifact@v4 + with: + name: durations-default-${{ strategy.job-index }} + path: durations.json + if-no-files-found: ignore + overwrite: true - # Oracle is left out of the job above: its tests run only with ORACLE_TESTS=true. This job runs what Oracle adds, - # on the newest Python alone: the tests whose name or parameters say oracle, which `-k oracle` picks. - test-oracle: - name: test (Oracle) + # What Oracle adds, on the newest Python alone, with ORACLE_TESTS=true: the tests whose name or parameters say oracle, + # which `-k oracle` picks. In four parts, and the nightly ones in a fifth on a full run. + oracle-part: + name: test (Oracle, ${{ matrix.part }}) runs-on: ubuntu-latest timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + part: ${{ fromJSON((inputs.full || github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'full-ci')) && '["1/4", "2/4", "3/4", "4/4", "nightly"]' || '["1/4", "2/4", "3/4", "4/4"]') }} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 @@ -93,6 +135,29 @@ jobs: run: docker pull gvenzl/oracle-free:23-faststart - run: uv sync - name: Run the Oracle tests - run: uv run pytest -q -k oracle env: ORACLE_TESTS: "true" + PART: ${{ matrix.part }} + run: | + if [ "$PART" = nightly ]; then picked=(-m nightly); else picked=(-m "not nightly" --shard "$PART"); fi + uv run pytest -q -k oracle --durations=15 --record-durations durations.json "${picked[@]}" + - if: always() + uses: actions/upload-artifact@v4 + with: + name: durations-oracle-${{ strategy.job-index }} + path: durations.json + if-no-files-found: ignore + overwrite: true + + # The check the single Oracle job gave before it was split: it passes when every part with Oracle passed. It runs + # whatever they concluded, as no job here may be skipped (see `tree`). + oracle: + name: test (Oracle) + needs: oracle-part + if: always() + runs-on: ubuntu-latest + steps: + - name: Every part passed + run: | + echo "The parts with Oracle concluded ${{ needs.oracle-part.result }}." + [ "${{ needs.oracle-part.result }}" = success ] diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..70b76433 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,97 @@ +# Each night, the full CI on main: every test, the nightly ones a pull request leaves out among them (`-m nightly`), on +# every store and every Python, and on the engines' latest images, without ci.yml's FalkorDB pin, so that an engine's +# new release shows up here and not in a pull request's run. At 17:00 UTC, 03:00 in Brisbane. +# +# A failure opens an issue labelled `nightly`, or comments on the one open, naming the failed jobs, their failing +# tests, and the commits since the last nightly that passed, among which is the one that broke it. A pass closes it. +# A run on a branch other than main writes what it would say to its summary instead. +name: Nightly + +on: + schedule: + - cron: "0 17 * * *" + workflow_dispatch: + +# Read only, so the tests run with a token that can't write to the repository; the report job asks for what it needs. +permissions: + contents: read + +jobs: + ci: + uses: ./.github/workflows/ci.yml + with: + full: true + + report: + needs: ci + if: always() + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + issues: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Open, update or close the nightly issue + env: + GH_TOKEN: ${{ github.token }} + RESULT: ${{ needs.ci.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + POST: ${{ github.ref == 'refs/heads/main' }} + run: | + repo="$GITHUB_REPOSITORY" + open=$(gh issue list --repo "$repo" --label nightly --state open --json number -q '.[0].number') + # Writes to the issue on main, and to the summary alone elsewhere. + post() { + echo "$1" + echo "$1" >> "$GITHUB_STEP_SUMMARY" + [ "$POST" = true ] || return 0 + if [ "$2" = close ]; then + gh issue close "$open" --repo "$repo" --comment "$1" + elif [ -n "$open" ]; then + gh issue comment "$open" --repo "$repo" --body "$1" + else + gh label create nightly --repo "$repo" --color B60205 --description "The nightly CI failed" 2> /dev/null || true + gh issue create --repo "$repo" --label nightly --title "Nightly CI fails" --body "$1" + fi + } + # A run cancelled by hand says nothing of the tree. + [ "$RESULT" != cancelled ] || exit 0 + if [ "$RESULT" = success ]; then + [ -z "$open" ] || post "The nightly passed again on $GITHUB_SHA: $RUN_URL" close + exit 0 + fi + body=$(mktemp) + { + echo "The nightly on $GITHUB_SHA concluded $RESULT: $RUN_URL" + echo + gh api --paginate "repos/$repo/actions/runs/$GITHUB_RUN_ID/jobs?per_page=100" \ + -q '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out") | "\(.id)\t\(.name)\t\(.html_url)"' | + while IFS=$'\t' read -r id name url; do + echo "### [$name]($url)" + echo + # pytest's short summary, a line for each failed test and each error, without the log's timestamps. + failing=$(gh api --allow-escape-sequences "repos/$repo/actions/jobs/$id/logs" | + sed -E 's/^[^ ]+ //; s/\x1b\[[0-9;]*m//g' | grep -E '^(FAILED|ERROR) ' | + cut -c1-300 | awk '!seen[$0]++' | head -40 || true) + if [ -n "$failing" ]; then + printf '```\n%s\n```\n' "$failing" + else + echo "No failing test in its log: it failed in a step around the tests, or waits for jobs above." + fi + echo + done + green=$(gh api "repos/$repo/actions/workflows/nightly.yml/runs?branch=main&status=success&per_page=1" \ + -q '.workflow_runs[0].head_sha // empty') + if [ -n "$green" ] && git cat-file -e "$green^{commit}" 2> /dev/null; then + echo "### Commits since the last nightly that passed, on $green" + echo + git log --format='- %h %s (%an)' "$green..$GITHUB_SHA" + [ -n "$(git log --format=%h "$green..$GITHUB_SHA")" ] || echo "None: the same commit passed before, so look at the images and the runners." + else + echo "No nightly has passed on main yet, so there are no commits to compare with." + fi + } > "$body" + post "$(cat "$body")" diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index cfce2dae..a19b091a 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -181,7 +181,8 @@ jobs: echo "This PR was opened with GITHUB_TOKEN, so CI hasn't started on it. Push an empty commit to the branch, or close and reopen the PR, to start it." fi } > "$body" - url=$(gh pr create --repo "$REPO" --base "$DEFAULT_BRANCH" --head "$BRANCH" --title "Release $TAG" --body-file "$body") + # Labelled full-ci, so its CI runs the nightly tests too, and a release of its tree can trust that run. + url=$(gh pr create --repo "$REPO" --base "$DEFAULT_BRANCH" --head "$BRANCH" --title "Release $TAG" --body-file "$body" --label full-ci) echo "Opened $url" | tee -a "$GITHUB_STEP_SUMMARY" if [[ "$HAS_PR_TOKEN" != true ]]; then note="Opened with GITHUB_TOKEN, so CI doesn't run on it by itself. Push an empty commit to $BRANCH, or close and reopen the PR, to start it." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b9488e39..abb6004f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ # - A push to main that raises pyproject.toml's version (the merge of a "Release vX.Y.Z" PR) publishes it, and tags # the commit vX.Y.Z. A push to main that changes pyproject.toml without raising the version publishes nothing. # - A push of a tag vX.Y.Z, by hand, publishes too: the fallback. -# - Either way, the full CI runs, unless CI has already passed on the same tree; the distributions are built and +# - Either way, the full CI runs, the nightly tests with it, unless a full CI run has already passed on the same tree; the distributions are built and # checked, and those exact files are uploaded to PyPI with trusted publishing. # - workflow_dispatch, and a pull request that changes the release machinery, are dry runs: everything but the # publish, with the built files left as the run's artifact to inspect. @@ -95,9 +95,9 @@ jobs: echo "publish=$publish" >> "$GITHUB_OUTPUT" echo "tag=$tag" >> "$GITHUB_OUTPUT" - # Whether ci.yml has already passed on the git tree this run builds, from a push or a pull request, so the gate - # needn't run it again: the release PR's run, when it was up to date with main and squash merged, or the push run on - # the merge commit, which starts beside this one and is waited for. A run on this tree that fails fails the release. + # Whether ci.yml has already passed on the git tree this run builds, from a push or a pull request, with the nightly + # tests, so the gate needn't run it again: the release PR's run (labelled full-ci), when it was up to date with main + # and squash merged, or the push run on the merge commit, which starts beside this one and is waited for. A run on this tree that fails fails the release. # scripts/ci_trust.py says how a run's tree is known, and why a pull request's run can be believed; the step summary # links the run trusted, or says why none was. Skipped, as the gate is, on a push that releases nothing. trust: @@ -125,12 +125,15 @@ jobs: tree=$(git rev-parse 'HEAD^{tree}') python3 scripts/ci_trust.py "$GITHUB_REPOSITORY" "$tree" "$(git rev-parse HEAD)" "$PR_HEAD" - # The full CI, as ci.yml runs it on a push or a pull request, unless trust found it has already passed on this tree. - # A push that releases nothing stops at check: ci.yml runs on the same push to main by itself. + # The full CI, as ci.yml runs it on a push to main, with the nightly tests a pull request's run leaves out, unless + # trust found it has already passed on this tree. A push that releases nothing stops at check: ci.yml runs on the + # same push to main by itself. gate: needs: [check, trust] if: needs.trust.outputs.trusted != 'true' uses: ./.github/workflows/ci.yml + with: + full: true # Runs when the gate passed, or was skipped because trust found CI had passed on this tree. A failed or cancelled # check, trust or gate stops it, and so does a gate skipped for any other reason (a push that releases nothing). diff --git a/RELEASING.md b/RELEASING.md index ffbf3350..778106af 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -51,7 +51,9 @@ git push origin vX.Y.Z - A run counts when it is from a push or a pull request (the nightly, which runs without the pins, doesn't), its head commit has this tree, and every job of its latest attempt concluded success. A pull request run tests the pull request's merge ref, not its head, so `ci.yml`'s first job records the tree it checked out as an artifact, - `ci-tree-`, and a pull request run counts only when it recorded this tree as well. A run of a fork's + `ci-tree-`, and a pull request run counts only when it recorded this tree as well. A pull request run that + left out the nightly tests, as one not labelled `full-ci` does, records `ci-tree-pr-set-` and doesn't + count; "Prepare release" labels its PR `full-ci`. A run of a fork's pull request is believed for the same reason: its head has this tree, so the `ci.yml` it ran is the one being released. - When one has passed, `gate` is skipped. When one is still going, as the push run on the merge commit is when @@ -60,8 +62,8 @@ git push origin vX.Y.Z CI run has been re-run too, which `trust` then waits for or trusts. When none counts, or the wait runs out, or the API can't be asked, `gate` runs as before. - The run's summary links the run trusted, or says of each run it looked at why it didn't count. -- **`gate`** runs the repository's full CI, `ci.yml`, unchanged, unless `trust` found it has already passed on this - tree. A push to the default branch that releases nothing skips both, as `ci.yml` runs on that push anyway. +- **`gate`** runs the repository's full CI, `ci.yml` with `full: true`, the nightly tests with it, unless `trust` + found it has already passed on this tree. A push to the default branch that releases nothing skips both, as `ci.yml` runs on that push anyway. - **`build`** builds the release artifacts from the checked-out ref, checks them, runs a smoke test, and uploads them as the run's artifact. - **`publish`** runs on a release alone, in the `release` environment. It publishes the exact files `build` made. diff --git a/docs/11-testing.md b/docs/11-testing.md index 46716bd4..7deccd8c 100644 --- a/docs/11-testing.md +++ b/docs/11-testing.md @@ -17,11 +17,15 @@ release reaches your tests as soon as it is out; pin one with its `_TEST_I Drivine's own tests run on Oracle only with `ORACLE_TESTS=true`: its image is large, and the database wants two gigabytes. A run without it ends with the line `Oracle tests skipped: set ORACLE_TESTS=true to run them`. CI runs the -Oracle tests in a job of their own, with `pytest -k oracle`. The tests run on the plugin's own images, the latest Neo4j, +Oracle tests in jobs of their own, with `pytest -k oracle`. CI splits each run over parallel jobs, `pytest --shard i/n` +(tests/shards.py), and a pull request's run leaves out the tests marked `nightly`: the timings, Oracle's plans, and the +texts past 32 KB. They run each night (`.github/workflows/nightly.yml`, which opens an issue labelled `nightly` when +they fail), on each push to main, in a release's gate, and on a pull request labelled `full-ci`. A local `pytest` runs +them all. The tests run on the plugin's own images, the latest Neo4j, Memgraph and FalkorDB, so that an engine release that changes what Drivine reads or writes shows up at once. CI holds FalkorDB at a release that passes, with `FALKORDB_TEST_IMAGE` in `.github/workflows/ci.yml`: Docker Hub's `falkordb/falkordb:latest` is sometimes a 4.x patch release -([FalkorDB/FalkorDB#3551](https://github.com/FalkorDB/FalkorDB/issues/3551)). CI's nightly run leaves the pin out, so +([FalkorDB/FalkorDB#3551](https://github.com/FalkorDB/FalkorDB/issues/3551)). The nightly run leaves the pin out, so it shows when the pin can move or go, and runs `ghcr.io/falkordb/falkordb:latest` in place of Docker Hub's `latest`: Docker Hub's is whichever release FalkorDB pushed last, and that can be an older line's backport, as `v4.22.3` took it after `6.0.3`, while ghcr.io's is FalkorDB's newest release. Once Docker Hub's `latest` follows FalkorDB's newest line diff --git a/pyproject.toml b/pyproject.toml index ef10bb9d..cc1372e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ markers = [ "needs_exact_float_reads: fails on a SQLite that reads 1.5E300 from text inexactly (#376); xfail there, strictly", "needs_sqlite_3_53: fails on SQLite before 3.53 (#376); xfail there, strictly", "async_only: runs tasks concurrently; left out of the sync tests scripts/unasync.py writes", + "nightly: slow, or rests on a timing or a plan; a pull request's CI leaves it out, and the nightly, a push to main, a release and a PR labelled full-ci run it", ] [tool.ruff] diff --git a/scripts/ci_trust.py b/scripts/ci_trust.py index 3d3e49a4..46657fce 100644 --- a/scripts/ci_trust.py +++ b/scripts/ci_trust.py @@ -24,6 +24,9 @@ - a pull_request run tested its merge ref instead, which is the same tree only when the pull request was up to date with its base. ci.yml's first job records the tree it checked out as an artifact named `ci-tree-`, and a pull_request run counts only when it recorded this tree, besides its head commit having it; +- it ran the full set. A pull request's run leaves out the nightly tests (`-m nightly`) unless it is labelled + `full-ci`, and then records its tree as `ci-tree-pr-set-`, which doesn't count, at once and not once it ends. + A push run is always full; - it passed: concluded success, with every job of its latest attempt concluded success. A job skipped by a condition would leave part of the tree untested and the run still green, so a skipped job means it doesn't count. @@ -59,6 +62,8 @@ CI = ".github/workflows/ci.yml" EVENTS = ("push", "pull_request") RECORD = "ci-tree-" +PR_SET = "pr-set-" +"""After RECORD, in the record of a run of the pull request set alone, without the nightly tests.""" # They go into the API's paths, so each must be what it says: a repository's name, and git object names. REPOSITORY = re.compile(r"[\w.-]+/[\w.-]+", re.ASCII) OBJECT = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}") @@ -154,6 +159,10 @@ def look(state: str, reason: str) -> Look: if _tree_of(run) != tree: return look("ignored", f"its commit {sha} is another tree") done = run.get("status") == "completed" + if run.get("event") == "pull_request" and f"{PR_SET}{tree}" in trees: + return look( + "ignored", "it ran the pull request set alone, without the nightly tests, as one not labelled full-ci does" + ) if run.get("event") == "pull_request" and tree not in trees: if trees: return look("ignored", f"it tested its merge ref, tree {trees[0]}, as its base had moved on") diff --git a/tests/conftest.py b/tests/conftest.py index 77840bd1..ad3b7c91 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,17 +4,20 @@ import pytest +# CI's parts of a run (--shard) and the durations they are balanced by: see tests/shards.py. +pytest_plugins = ["tests.shards"] + # The tests run on the plugin's defaults, the latest Neo4j, Memgraph and FalkorDB, which a user's tests get too, so that # an engine release that changes what Drivine reads or writes shows up here at once. The Cypher spec (tests/cypher) is # written for a current Neo4j, as drivine4j runs it. NEO4J_TEST_IMAGE, MEMGRAPH_TEST_IMAGE or FALKORDB_TEST_IMAGE runs -# another. CI holds FalkorDB back with FALKORDB_TEST_IMAGE (.github/workflows/ci.yml); its nightly run takes +# another. CI holds FalkorDB back with FALKORDB_TEST_IMAGE (.github/workflows/ci.yml); the nightly run takes # ghcr.io/falkordb/falkordb:latest, as Docker Hub's latest can be an older line's backport. # The datasources the tests run on. Drivine's plugin starts a container for each and points these at it. The host is # a name that resolves nowhere, so that with the plugin off the tests fail rather than clear a database of yours. To # run against your own, set the datasource and USE_LOCAL_NEO4J=true, USE_LOCAL_MEMGRAPH=true, USE_LOCAL_FALKORDB=true or # USE_LOCAL_POSTGRES=true. -# Oracle runs only with ORACLE_TESTS=true: its image is large and the database wants two gigabytes. CI runs it in a job +# Oracle runs only with ORACLE_TESTS=true: its image is large and the database wants two gigabytes. CI runs it in jobs # of its own. os.environ.setdefault("DATABASE_TYPE", "NEO4J") os.environ.setdefault("DATABASE_HOST", "test-container.invalid") diff --git a/tests/durations.json b/tests/durations.json new file mode 100644 index 00000000..813cc0e6 --- /dev/null +++ b/tests/durations.json @@ -0,0 +1,332 @@ +{ + "default": { + "tests/docs/test_readme.py": 2.21, + "tests/integration/oracle/test_oracle_adaptive_plans.py": 2.21, + "tests/integration/oracle/test_oracle_connection.py": 8.1, + "tests/integration/oracle/test_oracle_era_functions.py": 3.68, + "tests/integration/oracle/test_oracle_era_parse.py": 2.21, + "tests/integration/oracle/test_oracle_graph_name.py": 1.47, + "tests/integration/oracle/test_oracle_indexed_filters.py": 3.68, + "tests/integration/oracle/test_oracle_keyed_label_plan.py": 7.36, + "tests/integration/oracle/test_oracle_label_text.py": 4.42, + "tests/integration/oracle/test_oracle_long_text_memory.py": 0.74, + "tests/integration/oracle/test_oracle_node_equality.py": 5.15, + "tests/integration/oracle/test_oracle_nulls_equal.py": 5.89, + "tests/integration/oracle/test_oracle_schema.py": 5.89, + "tests/integration/oracle/test_oracle_schema_catalog.py": 1.45, + "tests/integration/oracle/test_oracle_temporal_without_era.py": 10.63, + "tests/integration/oracle/test_oracle_zoned_list_sort.py": 0.97, + "tests/integration/postgres/test_compiled_write_keyed_read_time.py": 1.69, + "tests/integration/postgres/test_postgres_connection.py": 2.66, + "tests/integration/postgres/test_postgres_key_probe_plan.py": 0.75, + "tests/integration/postgres/test_postgres_nested_loop_settings.py": 1.13, + "tests/integration/postgres/test_postgres_store.py": 5.28, + "tests/integration/sql/test_compiled_write_loop_time.py": 3.77, + "tests/integration/sql/test_graph_tables_on_first_use.py": 2.26, + "tests/integration/sql/test_id_key_plans.py": 7.54, + "tests/integration/sql/test_property_key_index.py": 6.97, + "tests/integration/sql/test_schema_ddl.py": 1.71, + "tests/integration/sql/test_sql_fragments.py": 0.6, + "tests/integration/sql/test_sql_saves.py": 0.53, + "tests/integration/sql/test_sql_views.py": 0.3, + "tests/integration/sql/test_sql_writes.py": 0.36, + "tests/integration/sqlite/test_sqlite_connection.py": 2.73, + "tests/integration/sqlite/test_sqlite_store.py": 3.94, + "tests/integration/test_assembly.py": 1.97, + "tests/integration/test_bm25.py": 11.83, + "tests/integration/test_case_insensitive.py": 18.2, + "tests/integration/test_cypher_compiled.py": 227.93, + "tests/integration/test_cypher_set_based_delete.py": 4.15, + "tests/integration/test_cypher_spec.py": 68.6, + "tests/integration/test_cypher_write_loop.py": 0.63, + "tests/integration/test_falkordb.py": 0.21, + "tests/integration/test_grouped_reads.py": 3.37, + "tests/integration/test_index_advice.py": 2.38, + "tests/integration/test_landing_page_sample.py": 0.6, + "tests/integration/test_object_manager_batches_patches_edges.py": 4.82, + "tests/integration/test_object_manager_loading.py": 2.31, + "tests/integration/test_object_manager_saving.py": 33.28, + "tests/integration/test_persistence_manager.py": 1.09, + "tests/integration/test_query_dsl.py": 3.12, + "tests/integration/test_recursive_cascade.py": 3.49, + "tests/integration/test_schema.py": 6.98, + "tests/integration/test_schema_runtime.py": 7.38, + "tests/integration/test_search.py": 6.49, + "tests/integration/test_transactions.py": 0.81, + "tests/integration/test_vector_scores.py": 0.55, + "tests/sync/test_graph_tables_threads.py": 0.03, + "tests/sync/test_store_identity_race.py": 0.17, + "tests/sync/test_threads.py": 0.37, + "tests/sync_integration/oracle/test_oracle_adaptive_plans.py": 0.1, + "tests/sync_integration/oracle/test_oracle_connection.py": 0.33, + "tests/sync_integration/oracle/test_oracle_era_functions.py": 0.17, + "tests/sync_integration/oracle/test_oracle_era_parse.py": 0.1, + "tests/sync_integration/oracle/test_oracle_graph_name.py": 0.07, + "tests/sync_integration/oracle/test_oracle_indexed_filters.py": 0.17, + "tests/sync_integration/oracle/test_oracle_keyed_label_plan.py": 0.33, + "tests/sync_integration/oracle/test_oracle_label_text.py": 0.2, + "tests/sync_integration/oracle/test_oracle_long_text_memory.py": 0.03, + "tests/sync_integration/oracle/test_oracle_node_equality.py": 0.31, + "tests/sync_integration/oracle/test_oracle_nulls_equal.py": 0.76, + "tests/sync_integration/oracle/test_oracle_schema.py": 0.76, + "tests/sync_integration/oracle/test_oracle_schema_catalog.py": 0.65, + "tests/sync_integration/oracle/test_oracle_temporal_without_era.py": 4.77, + "tests/sync_integration/oracle/test_oracle_zoned_list_sort.py": 0.43, + "tests/sync_integration/postgres/test_compiled_write_keyed_read_time.py": 2.24, + "tests/sync_integration/postgres/test_postgres_connection.py": 5.28, + "tests/sync_integration/postgres/test_postgres_key_probe_plan.py": 0.96, + "tests/sync_integration/postgres/test_postgres_nested_loop_settings.py": 1.44, + "tests/sync_integration/postgres/test_postgres_store.py": 6.72, + "tests/sync_integration/sql/test_compiled_write_loop_time.py": 4.8, + "tests/sync_integration/sql/test_graph_tables_on_first_use.py": 0.96, + "tests/sync_integration/sql/test_id_key_plans.py": 9.59, + "tests/sync_integration/sql/test_property_key_index.py": 4.61, + "tests/sync_integration/sql/test_schema_ddl.py": 2.75, + "tests/sync_integration/sql/test_sql_fragments.py": 0.77, + "tests/sync_integration/sql/test_sql_saves.py": 0.31, + "tests/sync_integration/sql/test_sql_views.py": 0.26, + "tests/sync_integration/sql/test_sql_writes.py": 0.31, + "tests/sync_integration/sqlite/test_sqlite_connection.py": 1.21, + "tests/sync_integration/sqlite/test_sqlite_store.py": 1.21, + "tests/sync_integration/test_assembly.py": 0.64, + "tests/sync_integration/test_bm25.py": 6.51, + "tests/sync_integration/test_case_insensitive.py": 12.73, + "tests/sync_integration/test_cypher_compiled.py": 196.29, + "tests/sync_integration/test_cypher_set_based_delete.py": 3.65, + "tests/sync_integration/test_cypher_spec.py": 59.18, + "tests/sync_integration/test_cypher_write_loop.py": 0.5, + "tests/sync_integration/test_falkordb.py": 0.17, + "tests/sync_integration/test_grouped_reads.py": 2.53, + "tests/sync_integration/test_index_advice.py": 1.82, + "tests/sync_integration/test_landing_page_sample.py": 0.46, + "tests/sync_integration/test_object_manager_batches_patches_edges.py": 3.84, + "tests/sync_integration/test_object_manager_loading.py": 1.5, + "tests/sync_integration/test_object_manager_saving.py": 6.43, + "tests/sync_integration/test_persistence_manager.py": 0.66, + "tests/sync_integration/test_query_dsl.py": 2.21, + "tests/sync_integration/test_recursive_cascade.py": 3.11, + "tests/sync_integration/test_schema.py": 3.58, + "tests/sync_integration/test_schema_runtime.py": 6.49, + "tests/sync_integration/test_search.py": 6.9, + "tests/sync_integration/test_transactions.py": 0.59, + "tests/sync_integration/test_vector_scores.py": 0.56, + "tests/unit/test_assembly_lifecycle.py": 0.76, + "tests/unit/test_bm25.py": 0.01, + "tests/unit/test_cascade.py": 0.03, + "tests/unit/test_case_insensitive.py": 0.1, + "tests/unit/test_ci_trust.py": 0.71, + "tests/unit/test_collection_fields.py": 0.37, + "tests/unit/test_compiled_write_kept_rows.py": 0.1, + "tests/unit/test_configuration.py": 0.38, + "tests/unit/test_conflict_sample.py": 0.1, + "tests/unit/test_constraint_manager.py": 0.44, + "tests/unit/test_contention.py": 0.33, + "tests/unit/test_cypher_bc_years.py": 0.41, + "tests/unit/test_cypher_compiled_size.py": 9.34, + "tests/unit/test_cypher_grouped_labels.py": 0.25, + "tests/unit/test_cypher_kind_order.py": 0.05, + "tests/unit/test_cypher_node_equality.py": 1.14, + "tests/unit/test_cypher_parameters.py": 0.87, + "tests/unit/test_cypher_property_key_index.py": 0.1, + "tests/unit/test_cypher_reader.py": 0.79, + "tests/unit/test_cypher_refusals.py": 7.39, + "tests/unit/test_cypher_set_based_delete.py": 0.13, + "tests/unit/test_cypher_shorthand.py": 0.11, + "tests/unit/test_cypher_spec_loader.py": 0.05, + "tests/unit/test_cypher_undirected_join.py": 1.15, + "tests/unit/test_cypher_variables_read_in.py": 0.05, + "tests/unit/test_cypher_zoned_list_sort.py": 0.16, + "tests/unit/test_drivine_test_containers.py": 0.07, + "tests/unit/test_full_text_index_bound.py": 0.12, + "tests/unit/test_golden_statements.py": 0.01, + "tests/unit/test_graph_tables.py": 0.01, + "tests/unit/test_graph_tables_providers.py": 0.03, + "tests/unit/test_graph_tables_providers_sync.py": 0.01, + "tests/unit/test_graph_tables_sync.py": 0.02, + "tests/unit/test_model_file.py": 0.37, + "tests/unit/test_model_registry.py": 0.26, + "tests/unit/test_model_registry_cycles.py": 0.08, + "tests/unit/test_object_names.py": 0.02, + "tests/unit/test_optional_antlr.py": 0.03, + "tests/unit/test_oracle_adaptive_plans.py": 0.06, + "tests/unit/test_oracle_compare_text.py": 0.14, + "tests/unit/test_oracle_contains_text.py": 0.05, + "tests/unit/test_oracle_era_functions.py": 0.39, + "tests/unit/test_oracle_graph.py": 0.11, + "tests/unit/test_oracle_item_property.py": 0.4, + "tests/unit/test_oracle_json_table_transform.py": 0.09, + "tests/unit/test_oracle_one_row_plan.py": 0.16, + "tests/unit/test_oracle_optional_join.py": 0.03, + "tests/unit/test_oracle_query_renderer.py": 0.05, + "tests/unit/test_oracle_reverse_text.py": 0.01, + "tests/unit/test_oracle_schema_grammar.py": 0.04, + "tests/unit/test_oracle_search.py": 0.01, + "tests/unit/test_oracle_statement_cache.py": 0.01, + "tests/unit/test_oracle_stored_functions.py": 0.06, + "tests/unit/test_oracle_version.py": 0.03, + "tests/unit/test_postgres_backend.py": 0.02, + "tests/unit/test_postgres_key_probe.py": 0.04, + "tests/unit/test_postgres_nested_loop_plan.py": 0.07, + "tests/unit/test_postgres_spec_compiler.py": 0.04, + "tests/unit/test_query_renderer.py": 0.0, + "tests/unit/test_recorded_inventory.py": 0.01, + "tests/unit/test_release_version.py": 0.07, + "tests/unit/test_save_all_lock_order.py": 0.07, + "tests/unit/test_save_statement_builder.py": 0.34, + "tests/unit/test_schema_commands.py": 0.07, + "tests/unit/test_schema_ddl.py": 0.39, + "tests/unit/test_schema_ddl_spec.py": 0.3, + "tests/unit/test_schema_nothing_to_apply.py": 0.16, + "tests/unit/test_sqlite_cypher_compiler.py": 0.27, + "tests/unit/test_sqlite_cypher_functions.py": 0.39, + "tests/unit/test_sqlite_store.py": 4.88, + "tests/unit/test_store_identity_race.py": 13.18, + "tests/unit/test_stubs.py": 1.5, + "tests/unit/test_sync_up_to_date.py": 0.3, + "tests/unit/test_text_query.py": 5.31, + "tests/unit/test_text_query_counting.py": 1.84, + "tests/unit/test_transactions.py": 1.35, + "tests/unit/test_unasync.py": 0.18, + "tests/unit/test_vector_annotation_check.py": 0.99, + "tests/unit/test_vector_quantization_read.py": 0.63 + }, + "oracle": { + "tests/integration/oracle/test_oracle_adaptive_plans.py": 8.05, + "tests/integration/oracle/test_oracle_connection.py": 29.52, + "tests/integration/oracle/test_oracle_era_functions.py": 13.42, + "tests/integration/oracle/test_oracle_era_parse.py": 8.05, + "tests/integration/oracle/test_oracle_graph_name.py": 5.37, + "tests/integration/oracle/test_oracle_indexed_filters.py": 13.42, + "tests/integration/oracle/test_oracle_keyed_label_plan.py": 26.83, + "tests/integration/oracle/test_oracle_label_text.py": 16.1, + "tests/integration/oracle/test_oracle_long_text_memory.py": 2.68, + "tests/integration/oracle/test_oracle_node_equality.py": 18.78, + "tests/integration/oracle/test_oracle_nulls_equal.py": 21.47, + "tests/integration/oracle/test_oracle_schema.py": 21.47, + "tests/integration/oracle/test_oracle_schema_catalog.py": 14.89, + "tests/integration/oracle/test_oracle_temporal_without_era.py": 100.3, + "tests/integration/oracle/test_oracle_zoned_list_sort.py": 9.12, + "tests/integration/sql/test_compiled_write_loop_time.py": 11.4, + "tests/integration/sql/test_graph_tables_on_first_use.py": 6.84, + "tests/integration/sql/test_id_key_plans.py": 27.36, + "tests/integration/sql/test_property_key_index.py": 15.78, + "tests/integration/sql/test_schema_ddl.py": 16.87, + "tests/integration/sql/test_sql_fragments.py": 5.91, + "tests/integration/sql/test_sql_saves.py": 12.66, + "tests/integration/sql/test_sql_views.py": 8.44, + "tests/integration/sql/test_sql_writes.py": 5.94, + "tests/integration/test_bm25.py": 5.14, + "tests/integration/test_case_insensitive.py": 4.17, + "tests/integration/test_cypher_compiled.py": 143.46, + "tests/integration/test_cypher_set_based_delete.py": 4.91, + "tests/integration/test_cypher_spec.py": 107.05, + "tests/integration/test_cypher_write_loop.py": 3.29, + "tests/integration/test_grouped_reads.py": 1.99, + "tests/integration/test_index_advice.py": 1.33, + "tests/integration/test_landing_page_sample.py": 0.33, + "tests/integration/test_object_manager_batches_patches_edges.py": 5.98, + "tests/integration/test_object_manager_loading.py": 6.31, + "tests/integration/test_object_manager_saving.py": 12.06, + "tests/integration/test_persistence_manager.py": 1.04, + "tests/integration/test_query_dsl.py": 4.41, + "tests/integration/test_recursive_cascade.py": 7.25, + "tests/integration/test_schema.py": 9.67, + "tests/integration/test_schema_runtime.py": 25.75, + "tests/integration/test_search.py": 66.69, + "tests/integration/test_transactions.py": 14.82, + "tests/integration/test_vector_scores.py": 9.88, + "tests/sync/test_graph_tables_threads.py": 2.47, + "tests/sync/test_store_identity_race.py": 2.47, + "tests/sync/test_threads.py": 4.94, + "tests/sync_integration/oracle/test_oracle_adaptive_plans.py": 7.41, + "tests/sync_integration/oracle/test_oracle_connection.py": 24.7, + "tests/sync_integration/oracle/test_oracle_era_functions.py": 12.35, + "tests/sync_integration/oracle/test_oracle_era_parse.py": 7.41, + "tests/sync_integration/oracle/test_oracle_graph_name.py": 4.94, + "tests/sync_integration/oracle/test_oracle_indexed_filters.py": 7.04, + "tests/sync_integration/oracle/test_oracle_keyed_label_plan.py": 11.43, + "tests/sync_integration/oracle/test_oracle_label_text.py": 6.86, + "tests/sync_integration/oracle/test_oracle_long_text_memory.py": 1.14, + "tests/sync_integration/oracle/test_oracle_node_equality.py": 8.0, + "tests/sync_integration/oracle/test_oracle_nulls_equal.py": 8.0, + "tests/sync_integration/oracle/test_oracle_schema.py": 8.0, + "tests/sync_integration/oracle/test_oracle_schema_catalog.py": 6.86, + "tests/sync_integration/oracle/test_oracle_temporal_without_era.py": 83.02, + "tests/sync_integration/oracle/test_oracle_zoned_list_sort.py": 11.12, + "tests/sync_integration/sql/test_compiled_write_loop_time.py": 13.9, + "tests/sync_integration/sql/test_graph_tables_on_first_use.py": 2.78, + "tests/sync_integration/sql/test_id_key_plans.py": 33.36, + "tests/sync_integration/sql/test_property_key_index.py": 47.26, + "tests/sync_integration/sql/test_schema_ddl.py": 37.98, + "tests/sync_integration/sql/test_sql_fragments.py": 2.14, + "tests/sync_integration/sql/test_sql_saves.py": 4.29, + "tests/sync_integration/sql/test_sql_views.py": 3.06, + "tests/sync_integration/sql/test_sql_writes.py": 3.67, + "tests/sync_integration/test_bm25.py": 4.9, + "tests/sync_integration/test_case_insensitive.py": 5.81, + "tests/sync_integration/test_cypher_compiled.py": 125.86, + "tests/sync_integration/test_cypher_set_based_delete.py": 4.79, + "tests/sync_integration/test_cypher_spec.py": 98.57, + "tests/sync_integration/test_cypher_write_loop.py": 4.95, + "tests/sync_integration/test_grouped_reads.py": 7.42, + "tests/sync_integration/test_index_advice.py": 4.95, + "tests/sync_integration/test_landing_page_sample.py": 1.24, + "tests/sync_integration/test_object_manager_batches_patches_edges.py": 22.27, + "tests/sync_integration/test_object_manager_loading.py": 2.96, + "tests/sync_integration/test_object_manager_saving.py": 3.75, + "tests/sync_integration/test_persistence_manager.py": 1.63, + "tests/sync_integration/test_query_dsl.py": 2.44, + "tests/sync_integration/test_recursive_cascade.py": 2.44, + "tests/sync_integration/test_schema.py": 4.38, + "tests/sync_integration/test_schema_runtime.py": 6.16, + "tests/sync_integration/test_search.py": 5.94, + "tests/sync_integration/test_transactions.py": 0.64, + "tests/sync_integration/test_vector_scores.py": 0.39, + "tests/unit/test_bm25.py": 0.39, + "tests/unit/test_cascade.py": 0.59, + "tests/unit/test_case_insensitive.py": 0.29, + "tests/unit/test_compiled_write_kept_rows.py": 0.2, + "tests/unit/test_conflict_sample.py": 0.2, + "tests/unit/test_contention.py": 0.29, + "tests/unit/test_cypher_bc_years.py": 0.2, + "tests/unit/test_cypher_compiled_size.py": 4.54, + "tests/unit/test_cypher_kind_order.py": 0.03, + "tests/unit/test_cypher_node_equality.py": 0.69, + "tests/unit/test_cypher_parameters.py": 0.23, + "tests/unit/test_cypher_property_key_index.py": 0.12, + "tests/unit/test_cypher_refusals.py": 3.56, + "tests/unit/test_cypher_set_based_delete.py": 0.3, + "tests/unit/test_cypher_undirected_join.py": 0.25, + "tests/unit/test_cypher_zoned_list_sort.py": 0.09, + "tests/unit/test_drivine_test_containers.py": 0.01, + "tests/unit/test_full_text_index_bound.py": 0.12, + "tests/unit/test_golden_statements.py": 0.01, + "tests/unit/test_graph_tables_providers.py": 0.13, + "tests/unit/test_graph_tables_providers_sync.py": 0.06, + "tests/unit/test_oracle_adaptive_plans.py": 0.06, + "tests/unit/test_oracle_compare_text.py": 0.1, + "tests/unit/test_oracle_contains_text.py": 0.03, + "tests/unit/test_oracle_era_functions.py": 0.25, + "tests/unit/test_oracle_graph.py": 0.07, + "tests/unit/test_oracle_item_property.py": 0.25, + "tests/unit/test_oracle_json_table_transform.py": 0.06, + "tests/unit/test_oracle_one_row_plan.py": 0.7, + "tests/unit/test_oracle_optional_join.py": 0.14, + "tests/unit/test_oracle_query_renderer.py": 0.28, + "tests/unit/test_oracle_reverse_text.py": 0.14, + "tests/unit/test_oracle_schema_grammar.py": 0.56, + "tests/unit/test_oracle_search.py": 0.14, + "tests/unit/test_oracle_statement_cache.py": 0.14, + "tests/unit/test_oracle_stored_functions.py": 0.98, + "tests/unit/test_oracle_version.py": 0.42, + "tests/unit/test_postgres_key_probe.py": 0.07, + "tests/unit/test_postgres_nested_loop_plan.py": 0.07, + "tests/unit/test_save_all_lock_order.py": 0.07, + "tests/unit/test_schema_ddl.py": 0.07, + "tests/unit/test_sqlite_cypher_compiler.py": 0.07, + "tests/unit/test_sqlite_store.py": 0.07, + "tests/unit/test_store_identity_race.py": 0.56, + "tests/unit/test_text_query.py": 3.62, + "tests/unit/test_text_query_counting.py": 1.02 + } +} diff --git a/tests/integration/oracle/test_oracle_adaptive_plans.py b/tests/integration/oracle/test_oracle_adaptive_plans.py index fa0c63ba..6f4d0e46 100644 --- a/tests/integration/oracle/test_oracle_adaptive_plans.py +++ b/tests/integration/oracle/test_oracle_adaptive_plans.py @@ -21,7 +21,7 @@ from tests.cypher.cypher_spec import load from tests.integration.oracle.oracle_seed import ON_ORACLE -pytestmark = ON_ORACLE +pytestmark = [ON_ORACLE, pytest.mark.nightly] GRAPH = "drivine_adaptive_plans" diff --git a/tests/integration/oracle/test_oracle_era_parse.py b/tests/integration/oracle/test_oracle_era_parse.py index 745f0462..626756f5 100644 --- a/tests/integration/oracle/test_oracle_era_parse.py +++ b/tests/integration/oracle/test_oracle_era_parse.py @@ -33,7 +33,7 @@ from tests.integration.oracle.oracle_seed import ON_ORACLE from tests.integration.oracle.test_oracle_nulls_equal import dropping -pytestmark = ON_ORACLE +pytestmark = [ON_ORACLE, pytest.mark.nightly] GRAPH = "cy_era_parse" """The graph the statements are compiled against: made first, with its functions, and dropped after.""" diff --git a/tests/integration/oracle/test_oracle_keyed_label_plan.py b/tests/integration/oracle/test_oracle_keyed_label_plan.py index 1f9dc1e5..0b769058 100644 --- a/tests/integration/oracle/test_oracle_keyed_label_plan.py +++ b/tests/integration/oracle/test_oracle_keyed_label_plan.py @@ -67,7 +67,7 @@ async def seeded(factory: PersistenceManagerFactory) -> AsyncIterator[None]: await clear(pm) -pytestmark = [ON_ORACLE, pytest.mark.usefixtures("seeded")] +pytestmark = [ON_ORACLE, pytest.mark.usefixtures("seeded"), pytest.mark.nightly] def later_compared(sql: str) -> str: diff --git a/tests/integration/oracle/test_oracle_long_text_memory.py b/tests/integration/oracle/test_oracle_long_text_memory.py index d2466201..602c2cc1 100644 --- a/tests/integration/oracle/test_oracle_long_text_memory.py +++ b/tests/integration/oracle/test_oracle_long_text_memory.py @@ -7,13 +7,14 @@ from typing import Any import oracledb +import pytest from drivine import ConnectionProperties, QuerySpecification, cypher_statement from drivine.connection.oracle.oracle_connection_provider import OracleConnectionProvider from drivine.testing.drivine_test_containers import TEST_CONTAINER_PASSWORD from tests.integration.oracle.oracle_seed import ON_ORACLE -pytestmark = ON_ORACLE +pytestmark = [ON_ORACLE, pytest.mark.nightly] GRAPH = "drivine_long_text_memory" diff --git a/tests/integration/oracle/test_oracle_node_equality.py b/tests/integration/oracle/test_oracle_node_equality.py index f66e8b63..b25d4ea2 100644 --- a/tests/integration/oracle/test_oracle_node_equality.py +++ b/tests/integration/oracle/test_oracle_node_equality.py @@ -42,7 +42,7 @@ async def seeded(factory: PersistenceManagerFactory) -> AsyncIterator[None]: await clear(pm) -pytestmark = [ON_ORACLE, pytest.mark.usefixtures("seeded")] +pytestmark = [ON_ORACLE, pytest.mark.usefixtures("seeded"), pytest.mark.nightly] async def assert_joined(pm: PersistenceManager, statement: str, expected: list[Any]) -> None: diff --git a/tests/integration/postgres/test_compiled_write_keyed_read_time.py b/tests/integration/postgres/test_compiled_write_keyed_read_time.py index a28a8a0b..32df1ac6 100644 --- a/tests/integration/postgres/test_compiled_write_keyed_read_time.py +++ b/tests/integration/postgres/test_compiled_write_keyed_read_time.py @@ -33,7 +33,7 @@ from tests.integration.postgres.conftest import ON_POSTGRES from tests.integration.sql.test_compiled_write_loop_time import LARGE, SMALL, slow -pytestmark = ON_POSTGRES +pytestmark = [ON_POSTGRES, pytest.mark.nightly] GRAPH = "cy_keyed" """The graph the writes are timed on: tables of their own, apart from those the other tests read.""" diff --git a/tests/integration/sql/test_compiled_write_loop_time.py b/tests/integration/sql/test_compiled_write_loop_time.py index f3b18cef..df29012c 100644 --- a/tests/integration/sql/test_compiled_write_loop_time.py +++ b/tests/integration/sql/test_compiled_write_loop_time.py @@ -29,6 +29,8 @@ from drivine import DatabaseType, PersistenceManager, QuerySpecification from drivine.query.statement import cypher_statement +pytestmark = pytest.mark.nightly + SMALL = 2_000 LARGE = 20_000 diff --git a/tests/integration/test_cypher_compiled.py b/tests/integration/test_cypher_compiled.py index 76ae12b1..571af57c 100644 --- a/tests/integration/test_cypher_compiled.py +++ b/tests/integration/test_cypher_compiled.py @@ -429,7 +429,7 @@ async def test_a_cypher_statement_is_compiled_against_the_tables_of_the_graph_it @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") -async def test_nodes_of_two_labels_a_transaction_has_made_and_not_yet_committed_are_found_by_both( +async def test_nodes_of_two_labels_a_transaction_has_made_and_not_yet_committed_are_found_by_both_on_oracle( factory: PersistenceManagerFactory, ): pm = factory.get("oracle") @@ -451,8 +451,9 @@ async def test_a_cypher_statement_that_writes_is_not_read_through_a_cursor_on_or await factory.get("oracle").open_cursor(cursor_spec) +@pytest.mark.nightly @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") -async def test_a_text_past_32_kb_set_on_two_nodes_beside_a_whole_float_or_not_leaves_the_session_open( +async def test_a_text_past_32_kb_set_on_two_nodes_beside_a_whole_float_or_not_leaves_the_session_open_on_oracle( factory: PersistenceManagerFactory, ): # Oracle crashed the session as the block freed its parameters, held in a JSON variable it had read once for each @@ -475,8 +476,9 @@ async def test_a_text_past_32_kb_set_on_two_nodes_beside_a_whole_float_or_not_le assert rows == [{"id": it, **written} for it in (1, 2)] +@pytest.mark.nightly @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") -async def test_a_text_past_32_kb_in_a_map_parameter_read_and_set_on_two_nodes_leaves_the_session_open( +async def test_a_text_past_32_kb_in_a_map_parameter_read_and_set_on_two_nodes_leaves_the_session_open_on_oracle( factory: PersistenceManagerFactory, ): # Oracle crashed the session where a statement read a value past 32 KB from a map parameter, by a property or by a @@ -505,6 +507,7 @@ async def test_a_text_past_32_kb_in_a_map_parameter_read_and_set_on_two_nodes_le assert rows == [{"id": it, **written} for it in (1, 2)] +@pytest.mark.nightly @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") async def test_a_property_past_32_kb_read_on_two_nodes_leaves_the_session_open_on_oracle( factory: PersistenceManagerFactory, registry: DatabaseRegistry @@ -558,6 +561,7 @@ async def query(cypher: str, parameters: dict[str, Any] | None = None) -> list[A await clear(factory.get("oracle")) +@pytest.mark.nightly @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") async def test_a_text_past_32_kb_set_from_an_unwound_parameter_on_two_nodes_leaves_the_session_open_on_oracle( factory: PersistenceManagerFactory, registry: DatabaseRegistry @@ -615,8 +619,9 @@ async def test_a_map_an_unwound_item_holds_with_a_key_as_extended_jsons_is_writt assert await pm.query(spec("MATCH (n:CyWrite) RETURN properties(n) ORDER BY n.id")) == items +@pytest.mark.nightly @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") -async def test_a_text_past_32_kb_from_an_unwound_parameter_returned_or_passed_on_over_two_rows_leaves_the_session_open( +async def test_a_text_past_32_kb_of_an_unwound_parameter_returned_or_passed_on_for_two_rows_leaves_oracles_session_open( factory: PersistenceManagerFactory, registry: DatabaseRegistry ): # Oracle crashed the session where a statement took a text past 32 KB from an item of an unwound parameter as JSON, diff --git a/tests/integration/test_object_manager_saving.py b/tests/integration/test_object_manager_saving.py index 1b652f9d..f42dda9e 100644 --- a/tests/integration/test_object_manager_saving.py +++ b/tests/integration/test_object_manager_saving.py @@ -882,6 +882,7 @@ async def bios(gom: StatelessGraphObjectManager) -> dict[str, str | None]: return {it.uuid: it.bio for it in await gom.load_all(Person)} +@pytest.mark.nightly @pytest.mark.parametrize("text", LONG_TEXTS, ids=["ascii", "multibyte"]) class TestLongText: async def test_saves_loads_and_saves_again_a_node_with_a_long_property(self, gom, text): diff --git a/tests/shards.py b/tests/shards.py new file mode 100644 index 00000000..bc6a016d --- /dev/null +++ b/tests/shards.py @@ -0,0 +1,104 @@ +""" +CI's parts of a test run: `pytest --shard 2/3` runs the second of three parts of what it would otherwise run, after `-k` +and `-m` have picked it, so that the parts of one selection, run on runners of their own, hold each of its tests once. +A module goes whole into one part, its module-scoped fixtures with it, in the order it is collected in. The parts are +balanced by how long each module took in CI, as tests/durations.json records it: under "oracle" with ORACLE_TESTS=true, +which runs what Oracle adds (`-k oracle`), and under "default" without. A module it doesn't name is reckoned at the +average time of a test in those it does, for each test it holds. + +`pytest --record-durations ` writes how long each module's tests took, setup and teardown with them, in seconds, +in the form of one of durations.json's two maps. CI uploads it from each part; merge the parts' files into +durations.json when the parts drift apart. +""" + +from __future__ import annotations + +import json +import os +from collections import defaultdict +from pathlib import Path + +import pytest + +DURATIONS = Path(__file__).with_name("durations.json") + +Part = tuple[int, int] + + +def parse(text: str) -> Part: + """`i/n`, the part i of n, numbered from 1.""" + index, _, count = text.partition("/") + try: + part = (int(index), int(count)) + except ValueError: + part = (0, 0) + if not 1 <= part[0] <= part[1]: + raise pytest.UsageError(f"--shard takes i/n, the part i of n parts, numbered from 1: not {text!r}.") + return part + + +def module_of(item: pytest.Item) -> str: + return item.nodeid.split("::", 1)[0] + + +def recorded() -> dict[str, float]: + """The seconds each module took, for the stores this run has.""" + if not DURATIONS.exists(): + return {} + key = "oracle" if os.environ.get("ORACLE_TESTS") == "true" else "default" + return json.loads(DURATIONS.read_text()).get(key, {}) + + +def parts(counts: dict[str, int], seconds: dict[str, float], count: int) -> list[set[str]]: + """ + The modules, by how many of their tests are to run, put in `count` parts of about the same time: the longest + first, each into the part with least so far, ties going to the lower part and the module named first. + """ + known = {module: seconds[module] for module in counts if module in seconds} + tested = sum(counts[module] for module in known) + average = sum(known.values()) / tested if tested else 1.0 + weight = {module: known.get(module, average * counts[module]) for module in counts} + totals = [0.0] * count + split: list[set[str]] = [set() for _ in range(count)] + for module in sorted(counts, key=lambda it: (-weight[it], it)): + lightest = min(range(count), key=lambda it: (totals[it], it)) + totals[lightest] += weight[module] + split[lightest].add(module) + return split + + +def pytest_addoption(parser: pytest.Parser) -> None: + group = parser.getgroup("drivine CI parts") + group.addoption("--shard", default=None, help="Run the part i of n of the tests picked, as i/n.") + group.addoption("--record-durations", default=None, help="Write the seconds each module's tests took to a file.") + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Keeps the tests of this part, after -k and -m, whose hooks run before this one, have deselected theirs.""" + text = config.getoption("--shard") + if not text: + return + index, count = parse(text) + counts: dict[str, int] = defaultdict(int) + for item in items: + counts[module_of(item)] += 1 + mine = parts(counts, recorded(), count)[index - 1] + kept = [it for it in items if module_of(it) in mine] + left = [it for it in items if module_of(it) not in mine] + if left: + config.hook.pytest_deselected(items=left) + items[:] = kept + + +TOOK: dict[str, float] = defaultdict(float) + + +def pytest_runtest_logreport(report: pytest.TestReport) -> None: + TOOK[report.nodeid.split("::", 1)[0]] += report.duration + + +def pytest_sessionfinish(session: pytest.Session) -> None: + path = session.config.getoption("--record-durations") + if path: + Path(path).write_text(json.dumps({it: round(TOOK[it], 2) for it in sorted(TOOK)}, indent=1) + "\n") diff --git a/tests/sync_integration/oracle/test_oracle_adaptive_plans.py b/tests/sync_integration/oracle/test_oracle_adaptive_plans.py index f21763cb..8b34adb8 100644 --- a/tests/sync_integration/oracle/test_oracle_adaptive_plans.py +++ b/tests/sync_integration/oracle/test_oracle_adaptive_plans.py @@ -29,7 +29,7 @@ from tests.cypher.cypher_spec import load from tests.sync_integration.oracle.oracle_seed import ON_ORACLE -pytestmark = ON_ORACLE +pytestmark = [ON_ORACLE, pytest.mark.nightly] GRAPH = "drivine_adaptive_plans" diff --git a/tests/sync_integration/oracle/test_oracle_era_parse.py b/tests/sync_integration/oracle/test_oracle_era_parse.py index 4e8b4ee1..474da8c0 100644 --- a/tests/sync_integration/oracle/test_oracle_era_parse.py +++ b/tests/sync_integration/oracle/test_oracle_era_parse.py @@ -35,7 +35,7 @@ from tests.sync_integration.oracle.oracle_seed import ON_ORACLE from tests.sync_integration.oracle.test_oracle_nulls_equal import dropping -pytestmark = ON_ORACLE +pytestmark = [ON_ORACLE, pytest.mark.nightly] GRAPH = "cy_era_parse" """The graph the statements are compiled against: made first, with its functions, and dropped after.""" diff --git a/tests/sync_integration/oracle/test_oracle_keyed_label_plan.py b/tests/sync_integration/oracle/test_oracle_keyed_label_plan.py index 2ac126b7..347768eb 100644 --- a/tests/sync_integration/oracle/test_oracle_keyed_label_plan.py +++ b/tests/sync_integration/oracle/test_oracle_keyed_label_plan.py @@ -65,7 +65,7 @@ def seeded(factory: PersistenceManagerFactory) -> Iterator[None]: clear(pm) -pytestmark = [ON_ORACLE, pytest.mark.usefixtures("seeded")] +pytestmark = [ON_ORACLE, pytest.mark.usefixtures("seeded"), pytest.mark.nightly] def later_compared(sql: str) -> str: diff --git a/tests/sync_integration/oracle/test_oracle_long_text_memory.py b/tests/sync_integration/oracle/test_oracle_long_text_memory.py index dc1f10a9..e248d82e 100644 --- a/tests/sync_integration/oracle/test_oracle_long_text_memory.py +++ b/tests/sync_integration/oracle/test_oracle_long_text_memory.py @@ -9,13 +9,14 @@ from typing import Any import oracledb +import pytest from drivine.sync import ConnectionProperties, QuerySpecification, cypher_statement from drivine.sync.connection.oracle.oracle_connection_provider import OracleConnectionProvider from drivine.testing.drivine_test_containers import TEST_CONTAINER_PASSWORD from tests.sync_integration.oracle.oracle_seed import ON_ORACLE -pytestmark = ON_ORACLE +pytestmark = [ON_ORACLE, pytest.mark.nightly] GRAPH = "drivine_long_text_memory" diff --git a/tests/sync_integration/oracle/test_oracle_node_equality.py b/tests/sync_integration/oracle/test_oracle_node_equality.py index e3fa4466..b30a57bd 100644 --- a/tests/sync_integration/oracle/test_oracle_node_equality.py +++ b/tests/sync_integration/oracle/test_oracle_node_equality.py @@ -44,7 +44,7 @@ def seeded(factory: PersistenceManagerFactory) -> Iterator[None]: clear(pm) -pytestmark = [ON_ORACLE, pytest.mark.usefixtures("seeded")] +pytestmark = [ON_ORACLE, pytest.mark.usefixtures("seeded"), pytest.mark.nightly] def assert_joined(pm: PersistenceManager, statement: str, expected: list[Any]) -> None: diff --git a/tests/sync_integration/postgres/test_compiled_write_keyed_read_time.py b/tests/sync_integration/postgres/test_compiled_write_keyed_read_time.py index 1b9134e0..fe836bec 100644 --- a/tests/sync_integration/postgres/test_compiled_write_keyed_read_time.py +++ b/tests/sync_integration/postgres/test_compiled_write_keyed_read_time.py @@ -35,7 +35,7 @@ from tests.sync_integration.postgres.conftest import ON_POSTGRES from tests.sync_integration.sql.test_compiled_write_loop_time import LARGE, SMALL, slow -pytestmark = ON_POSTGRES +pytestmark = [ON_POSTGRES, pytest.mark.nightly] GRAPH = "cy_keyed" """The graph the writes are timed on: tables of their own, apart from those the other tests read.""" diff --git a/tests/sync_integration/sql/test_compiled_write_loop_time.py b/tests/sync_integration/sql/test_compiled_write_loop_time.py index 984238ac..691e47d4 100644 --- a/tests/sync_integration/sql/test_compiled_write_loop_time.py +++ b/tests/sync_integration/sql/test_compiled_write_loop_time.py @@ -31,6 +31,8 @@ from drivine.query.statement import cypher_statement from drivine.sync import DatabaseType, PersistenceManager, QuerySpecification +pytestmark = pytest.mark.nightly + SMALL = 2_000 LARGE = 20_000 diff --git a/tests/sync_integration/test_cypher_compiled.py b/tests/sync_integration/test_cypher_compiled.py index dbf27975..f2a3325f 100644 --- a/tests/sync_integration/test_cypher_compiled.py +++ b/tests/sync_integration/test_cypher_compiled.py @@ -429,7 +429,7 @@ def test_a_cypher_statement_is_compiled_against_the_tables_of_the_graph_its_data @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") -def test_nodes_of_two_labels_a_transaction_has_made_and_not_yet_committed_are_found_by_both( +def test_nodes_of_two_labels_a_transaction_has_made_and_not_yet_committed_are_found_by_both_on_oracle( factory: PersistenceManagerFactory, ): pm = factory.get("oracle") @@ -451,8 +451,9 @@ def test_a_cypher_statement_that_writes_is_not_read_through_a_cursor_on_oracle( factory.get("oracle").open_cursor(cursor_spec) +@pytest.mark.nightly @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") -def test_a_text_past_32_kb_set_on_two_nodes_beside_a_whole_float_or_not_leaves_the_session_open( +def test_a_text_past_32_kb_set_on_two_nodes_beside_a_whole_float_or_not_leaves_the_session_open_on_oracle( factory: PersistenceManagerFactory, ): # Oracle crashed the session as the block freed its parameters, held in a JSON variable it had read once for each @@ -473,8 +474,9 @@ def test_a_text_past_32_kb_set_on_two_nodes_beside_a_whole_float_or_not_leaves_t assert rows == [{"id": it, **written} for it in (1, 2)] +@pytest.mark.nightly @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") -def test_a_text_past_32_kb_in_a_map_parameter_read_and_set_on_two_nodes_leaves_the_session_open( +def test_a_text_past_32_kb_in_a_map_parameter_read_and_set_on_two_nodes_leaves_the_session_open_on_oracle( factory: PersistenceManagerFactory, ): # Oracle crashed the session where a statement read a value past 32 KB from a map parameter, by a property or by a @@ -503,6 +505,7 @@ def test_a_text_past_32_kb_in_a_map_parameter_read_and_set_on_two_nodes_leaves_t assert rows == [{"id": it, **written} for it in (1, 2)] +@pytest.mark.nightly @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") def test_a_property_past_32_kb_read_on_two_nodes_leaves_the_session_open_on_oracle( factory: PersistenceManagerFactory, registry: DatabaseRegistry @@ -556,6 +559,7 @@ def query(cypher: str, parameters: dict[str, Any] | None = None) -> list[Any]: clear(factory.get("oracle")) +@pytest.mark.nightly @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") def test_a_text_past_32_kb_set_from_an_unwound_parameter_on_two_nodes_leaves_the_session_open_on_oracle( factory: PersistenceManagerFactory, registry: DatabaseRegistry @@ -613,8 +617,9 @@ def test_a_map_an_unwound_item_holds_with_a_key_as_extended_jsons_is_written_as_ assert pm.query(spec("MATCH (n:CyWrite) RETURN properties(n) ORDER BY n.id")) == items +@pytest.mark.nightly @pytest.mark.skipif(not ORACLE, reason="Oracle isn't configured: set ORACLE_TESTS=true.") -def test_a_text_past_32_kb_from_an_unwound_parameter_returned_or_passed_on_over_two_rows_leaves_the_session_open( +def test_a_text_past_32_kb_of_an_unwound_parameter_returned_or_passed_on_for_two_rows_leaves_oracles_session_open( factory: PersistenceManagerFactory, registry: DatabaseRegistry ): # Oracle crashed the session where a statement took a text past 32 KB from an item of an unwound parameter as JSON, diff --git a/tests/sync_integration/test_object_manager_saving.py b/tests/sync_integration/test_object_manager_saving.py index 5d624e1a..96d6fcf5 100644 --- a/tests/sync_integration/test_object_manager_saving.py +++ b/tests/sync_integration/test_object_manager_saving.py @@ -821,6 +821,7 @@ def bios(gom: StatelessGraphObjectManager) -> dict[str, str | None]: return {it.uuid: it.bio for it in gom.load_all(Person)} +@pytest.mark.nightly @pytest.mark.parametrize("text", LONG_TEXTS, ids=["ascii", "multibyte"]) class TestLongText: def test_saves_loads_and_saves_again_a_node_with_a_long_property(self, gom, text): diff --git a/tests/unit/test_ci_trust.py b/tests/unit/test_ci_trust.py index 422e6425..f540c597 100644 --- a/tests/unit/test_ci_trust.py +++ b/tests/unit/test_ci_trust.py @@ -219,6 +219,23 @@ def test_doesnt_trust_a_pull_requests_run_that_recorded_no_tree_as_one_before_ci assert "it recorded no tree" in result.text +def test_doesnt_wait_for_or_trust_a_pull_requests_run_of_the_pr_set_alone_as_it_leaves_out_the_nightly_tests() -> None: + answers = { + runs_on(SHA): workflow_runs(), + runs_on(HEAD): workflow_runs(going(11, event="pull_request", head_sha=HEAD)), + RECORDS: record_on(), + artifacts_of(11): recorded(f"pr-set-{TREE}"), + } + going_run = decide(answers, [SHA, HEAD]) + answers[runs_on(HEAD)] = workflow_runs(run(11, event="pull_request", head_sha=HEAD)) + answers[jobs_of(11)] = PASSED_JOBS + passed = decide(answers, [SHA, HEAD]) + + assert (going_run.decision, going_run.sleeps) == ("gate", []) + assert passed.decision == "gate" + assert "it ran the pull request set alone, without the nightly tests" in passed.text + + def test_counts_neither_the_nightly_which_drops_the_falkordb_pin_nor_a_run_of_another_workflow() -> None: result = decide( { diff --git a/tests/unit/test_shards.py b/tests/unit/test_shards.py new file mode 100644 index 00000000..f07ab0d8 --- /dev/null +++ b/tests/unit/test_shards.py @@ -0,0 +1,37 @@ +"""tests/shards.py: CI's parts of a run, each module whole in one part, balanced by the time each took.""" + +import pytest + +from tests.shards import parse, parts + + +def test_each_module_goes_into_exactly_one_part_the_longest_first_into_the_part_with_least() -> None: + counts = {"a.py": 10, "b.py": 10, "c.py": 10, "d.py": 10} + seconds = {"a.py": 100.0, "b.py": 60.0, "c.py": 50.0, "d.py": 10.0} + + # a.py's 100 s first, then b.py and c.py together, 110, and d.py with a.py: 110 and 110. + assert parts(counts, seconds, 2) == [{"a.py", "d.py"}, {"b.py", "c.py"}] + + +def test_a_module_without_a_time_is_reckoned_at_the_average_test_of_those_with_one() -> None: + # 2 s a test in a.py, so b.py's 30 tests are reckoned at 60 s, more than a.py's 20. + assert parts({"a.py": 10, "b.py": 30, "c.py": 1}, {"a.py": 20.0, "c.py": 2.0}, 2) == [{"b.py"}, {"a.py", "c.py"}] + + +def test_the_parts_of_one_selection_are_the_same_whichever_order_it_was_collected_in() -> None: + counts = {f"m{it}.py": it % 7 + 1 for it in range(40)} + reversed_counts = dict(reversed(list(counts.items()))) + + split = parts(counts, {}, 3) + assert split == parts(reversed_counts, {}, 3) + assert set().union(*split) == set(counts) and sum(len(it) for it in split) == len(counts) + + +@pytest.mark.parametrize("text", ["0/3", "4/3", "1", "a/b", "1/0"]) +def test_a_part_is_i_of_n_numbered_from_1(text: str) -> None: + with pytest.raises(pytest.UsageError, match="--shard takes i/n"): + parse(text) + + +def test_parses_i_of_n() -> None: + assert parse("2/3") == (2, 3) From a069eaa344eaddd71821080aaf1d2b82bc730273 Mon Sep 17 00:00:00 2001 From: jasper blues Date: Sun, 8 Nov 2026 14:31:53 +1000 Subject: [PATCH 2/5] Balance CI's parts by the durations the first split run recorded --- tests/durations.json | 621 +++++++++++++++++++++---------------------- 1 file changed, 300 insertions(+), 321 deletions(-) diff --git a/tests/durations.json b/tests/durations.json index 813cc0e6..63f07e89 100644 --- a/tests/durations.json +++ b/tests/durations.json @@ -1,332 +1,311 @@ { "default": { - "tests/docs/test_readme.py": 2.21, - "tests/integration/oracle/test_oracle_adaptive_plans.py": 2.21, - "tests/integration/oracle/test_oracle_connection.py": 8.1, - "tests/integration/oracle/test_oracle_era_functions.py": 3.68, - "tests/integration/oracle/test_oracle_era_parse.py": 2.21, - "tests/integration/oracle/test_oracle_graph_name.py": 1.47, - "tests/integration/oracle/test_oracle_indexed_filters.py": 3.68, - "tests/integration/oracle/test_oracle_keyed_label_plan.py": 7.36, - "tests/integration/oracle/test_oracle_label_text.py": 4.42, - "tests/integration/oracle/test_oracle_long_text_memory.py": 0.74, - "tests/integration/oracle/test_oracle_node_equality.py": 5.15, - "tests/integration/oracle/test_oracle_nulls_equal.py": 5.89, - "tests/integration/oracle/test_oracle_schema.py": 5.89, - "tests/integration/oracle/test_oracle_schema_catalog.py": 1.45, - "tests/integration/oracle/test_oracle_temporal_without_era.py": 10.63, - "tests/integration/oracle/test_oracle_zoned_list_sort.py": 0.97, - "tests/integration/postgres/test_compiled_write_keyed_read_time.py": 1.69, - "tests/integration/postgres/test_postgres_connection.py": 2.66, - "tests/integration/postgres/test_postgres_key_probe_plan.py": 0.75, - "tests/integration/postgres/test_postgres_nested_loop_settings.py": 1.13, - "tests/integration/postgres/test_postgres_store.py": 5.28, - "tests/integration/sql/test_compiled_write_loop_time.py": 3.77, - "tests/integration/sql/test_graph_tables_on_first_use.py": 2.26, - "tests/integration/sql/test_id_key_plans.py": 7.54, - "tests/integration/sql/test_property_key_index.py": 6.97, - "tests/integration/sql/test_schema_ddl.py": 1.71, - "tests/integration/sql/test_sql_fragments.py": 0.6, - "tests/integration/sql/test_sql_saves.py": 0.53, - "tests/integration/sql/test_sql_views.py": 0.3, - "tests/integration/sql/test_sql_writes.py": 0.36, - "tests/integration/sqlite/test_sqlite_connection.py": 2.73, - "tests/integration/sqlite/test_sqlite_store.py": 3.94, - "tests/integration/test_assembly.py": 1.97, - "tests/integration/test_bm25.py": 11.83, - "tests/integration/test_case_insensitive.py": 18.2, - "tests/integration/test_cypher_compiled.py": 227.93, - "tests/integration/test_cypher_set_based_delete.py": 4.15, - "tests/integration/test_cypher_spec.py": 68.6, - "tests/integration/test_cypher_write_loop.py": 0.63, - "tests/integration/test_falkordb.py": 0.21, - "tests/integration/test_grouped_reads.py": 3.37, - "tests/integration/test_index_advice.py": 2.38, - "tests/integration/test_landing_page_sample.py": 0.6, - "tests/integration/test_object_manager_batches_patches_edges.py": 4.82, - "tests/integration/test_object_manager_loading.py": 2.31, - "tests/integration/test_object_manager_saving.py": 33.28, - "tests/integration/test_persistence_manager.py": 1.09, - "tests/integration/test_query_dsl.py": 3.12, - "tests/integration/test_recursive_cascade.py": 3.49, - "tests/integration/test_schema.py": 6.98, - "tests/integration/test_schema_runtime.py": 7.38, - "tests/integration/test_search.py": 6.49, - "tests/integration/test_transactions.py": 0.81, - "tests/integration/test_vector_scores.py": 0.55, - "tests/sync/test_graph_tables_threads.py": 0.03, - "tests/sync/test_store_identity_race.py": 0.17, - "tests/sync/test_threads.py": 0.37, - "tests/sync_integration/oracle/test_oracle_adaptive_plans.py": 0.1, - "tests/sync_integration/oracle/test_oracle_connection.py": 0.33, - "tests/sync_integration/oracle/test_oracle_era_functions.py": 0.17, - "tests/sync_integration/oracle/test_oracle_era_parse.py": 0.1, - "tests/sync_integration/oracle/test_oracle_graph_name.py": 0.07, - "tests/sync_integration/oracle/test_oracle_indexed_filters.py": 0.17, - "tests/sync_integration/oracle/test_oracle_keyed_label_plan.py": 0.33, - "tests/sync_integration/oracle/test_oracle_label_text.py": 0.2, - "tests/sync_integration/oracle/test_oracle_long_text_memory.py": 0.03, - "tests/sync_integration/oracle/test_oracle_node_equality.py": 0.31, - "tests/sync_integration/oracle/test_oracle_nulls_equal.py": 0.76, - "tests/sync_integration/oracle/test_oracle_schema.py": 0.76, - "tests/sync_integration/oracle/test_oracle_schema_catalog.py": 0.65, - "tests/sync_integration/oracle/test_oracle_temporal_without_era.py": 4.77, - "tests/sync_integration/oracle/test_oracle_zoned_list_sort.py": 0.43, - "tests/sync_integration/postgres/test_compiled_write_keyed_read_time.py": 2.24, - "tests/sync_integration/postgres/test_postgres_connection.py": 5.28, - "tests/sync_integration/postgres/test_postgres_key_probe_plan.py": 0.96, - "tests/sync_integration/postgres/test_postgres_nested_loop_settings.py": 1.44, - "tests/sync_integration/postgres/test_postgres_store.py": 6.72, - "tests/sync_integration/sql/test_compiled_write_loop_time.py": 4.8, - "tests/sync_integration/sql/test_graph_tables_on_first_use.py": 0.96, - "tests/sync_integration/sql/test_id_key_plans.py": 9.59, - "tests/sync_integration/sql/test_property_key_index.py": 4.61, - "tests/sync_integration/sql/test_schema_ddl.py": 2.75, - "tests/sync_integration/sql/test_sql_fragments.py": 0.77, - "tests/sync_integration/sql/test_sql_saves.py": 0.31, - "tests/sync_integration/sql/test_sql_views.py": 0.26, - "tests/sync_integration/sql/test_sql_writes.py": 0.31, - "tests/sync_integration/sqlite/test_sqlite_connection.py": 1.21, - "tests/sync_integration/sqlite/test_sqlite_store.py": 1.21, - "tests/sync_integration/test_assembly.py": 0.64, - "tests/sync_integration/test_bm25.py": 6.51, - "tests/sync_integration/test_case_insensitive.py": 12.73, - "tests/sync_integration/test_cypher_compiled.py": 196.29, - "tests/sync_integration/test_cypher_set_based_delete.py": 3.65, - "tests/sync_integration/test_cypher_spec.py": 59.18, - "tests/sync_integration/test_cypher_write_loop.py": 0.5, - "tests/sync_integration/test_falkordb.py": 0.17, - "tests/sync_integration/test_grouped_reads.py": 2.53, - "tests/sync_integration/test_index_advice.py": 1.82, - "tests/sync_integration/test_landing_page_sample.py": 0.46, - "tests/sync_integration/test_object_manager_batches_patches_edges.py": 3.84, - "tests/sync_integration/test_object_manager_loading.py": 1.5, - "tests/sync_integration/test_object_manager_saving.py": 6.43, - "tests/sync_integration/test_persistence_manager.py": 0.66, - "tests/sync_integration/test_query_dsl.py": 2.21, - "tests/sync_integration/test_recursive_cascade.py": 3.11, - "tests/sync_integration/test_schema.py": 3.58, - "tests/sync_integration/test_schema_runtime.py": 6.49, - "tests/sync_integration/test_search.py": 6.9, - "tests/sync_integration/test_transactions.py": 0.59, - "tests/sync_integration/test_vector_scores.py": 0.56, - "tests/unit/test_assembly_lifecycle.py": 0.76, - "tests/unit/test_bm25.py": 0.01, - "tests/unit/test_cascade.py": 0.03, - "tests/unit/test_case_insensitive.py": 0.1, - "tests/unit/test_ci_trust.py": 0.71, - "tests/unit/test_collection_fields.py": 0.37, + "tests/docs/test_readme.py": 4.46, + "tests/integration/oracle/test_oracle_connection.py": 0.0, + "tests/integration/oracle/test_oracle_era_functions.py": 0.0, + "tests/integration/oracle/test_oracle_graph_name.py": 0.0, + "tests/integration/oracle/test_oracle_indexed_filters.py": 0.0, + "tests/integration/oracle/test_oracle_label_text.py": 0.0, + "tests/integration/oracle/test_oracle_nulls_equal.py": 0.0, + "tests/integration/oracle/test_oracle_schema.py": 0.0, + "tests/integration/oracle/test_oracle_schema_catalog.py": 0.0, + "tests/integration/oracle/test_oracle_temporal_without_era.py": 0.01, + "tests/integration/oracle/test_oracle_zoned_list_sort.py": 0.0, + "tests/integration/postgres/test_postgres_connection.py": 0.32, + "tests/integration/postgres/test_postgres_key_probe_plan.py": 5.62, + "tests/integration/postgres/test_postgres_nested_loop_settings.py": 0.31, + "tests/integration/postgres/test_postgres_non_finite.py": 0.31, + "tests/integration/postgres/test_postgres_store.py": 0.48, + "tests/integration/sql/test_graph_tables_on_first_use.py": 1.24, + "tests/integration/sql/test_id_key_plans.py": 2.08, + "tests/integration/sql/test_property_key_index.py": 6.49, + "tests/integration/sql/test_schema_ddl.py": 1.49, + "tests/integration/sql/test_sql_fragments.py": 0.08, + "tests/integration/sql/test_sql_saves.py": 0.17, + "tests/integration/sql/test_sql_views.py": 0.1, + "tests/integration/sql/test_sql_writes.py": 0.22, + "tests/integration/sqlite/test_sqlite_connection.py": 1.32, + "tests/integration/sqlite/test_sqlite_store.py": 0.99, + "tests/integration/test_assembly.py": 0.75, + "tests/integration/test_bm25.py": 26.85, + "tests/integration/test_case_insensitive.py": 9.4, + "tests/integration/test_cypher_compiled.py": 219.67, + "tests/integration/test_cypher_set_based_delete.py": 2.01, + "tests/integration/test_cypher_spec.py": 63.53, + "tests/integration/test_cypher_write_loop.py": 1.55, + "tests/integration/test_falkordb.py": 0.03, + "tests/integration/test_grouped_reads.py": 1.0, + "tests/integration/test_index_advice.py": 0.62, + "tests/integration/test_landing_page_sample.py": 1.0, + "tests/integration/test_object_manager_batches_patches_edges.py": 9.0, + "tests/integration/test_object_manager_loading.py": 2.73, + "tests/integration/test_object_manager_saving.py": 30.62, + "tests/integration/test_persistence_manager.py": 0.63, + "tests/integration/test_query_dsl.py": 2.54, + "tests/integration/test_recursive_cascade.py": 5.07, + "tests/integration/test_schema.py": 7.36, + "tests/integration/test_schema_runtime.py": 7.05, + "tests/integration/test_search.py": 7.15, + "tests/integration/test_transactions.py": 0.31, + "tests/integration/test_vector_scores.py": 0.64, + "tests/sync/test_graph_tables_threads.py": 0.07, + "tests/sync/test_store_identity_race.py": 2.51, + "tests/sync/test_threads.py": 0.11, + "tests/sync_integration/oracle/test_oracle_connection.py": 0.0, + "tests/sync_integration/oracle/test_oracle_era_functions.py": 0.0, + "tests/sync_integration/oracle/test_oracle_graph_name.py": 0.0, + "tests/sync_integration/oracle/test_oracle_indexed_filters.py": 0.0, + "tests/sync_integration/oracle/test_oracle_label_text.py": 0.0, + "tests/sync_integration/oracle/test_oracle_nulls_equal.py": 0.0, + "tests/sync_integration/oracle/test_oracle_schema.py": 0.0, + "tests/sync_integration/oracle/test_oracle_schema_catalog.py": 0.0, + "tests/sync_integration/oracle/test_oracle_temporal_without_era.py": 0.01, + "tests/sync_integration/oracle/test_oracle_zoned_list_sort.py": 0.0, + "tests/sync_integration/postgres/test_postgres_connection.py": 0.2, + "tests/sync_integration/postgres/test_postgres_key_probe_plan.py": 5.05, + "tests/sync_integration/postgres/test_postgres_nested_loop_settings.py": 0.04, + "tests/sync_integration/postgres/test_postgres_non_finite.py": 0.01, + "tests/sync_integration/postgres/test_postgres_store.py": 0.33, + "tests/sync_integration/sql/test_graph_tables_on_first_use.py": 0.04, + "tests/sync_integration/sql/test_id_key_plans.py": 1.63, + "tests/sync_integration/sql/test_property_key_index.py": 5.88, + "tests/sync_integration/sql/test_schema_ddl.py": 1.16, + "tests/sync_integration/sql/test_sql_fragments.py": 0.05, + "tests/sync_integration/sql/test_sql_saves.py": 0.34, + "tests/sync_integration/sql/test_sql_views.py": 0.17, + "tests/sync_integration/sql/test_sql_writes.py": 0.25, + "tests/sync_integration/sqlite/test_sqlite_connection.py": 0.11, + "tests/sync_integration/sqlite/test_sqlite_store.py": 0.24, + "tests/sync_integration/test_assembly.py": 0.11, + "tests/sync_integration/test_bm25.py": 18.44, + "tests/sync_integration/test_case_insensitive.py": 2.69, + "tests/sync_integration/test_cypher_compiled.py": 176.82, + "tests/sync_integration/test_cypher_set_based_delete.py": 0.87, + "tests/sync_integration/test_cypher_spec.py": 42.74, + "tests/sync_integration/test_cypher_write_loop.py": 0.81, + "tests/sync_integration/test_falkordb.py": 0.01, + "tests/sync_integration/test_grouped_reads.py": 0.63, + "tests/sync_integration/test_index_advice.py": 0.27, + "tests/sync_integration/test_landing_page_sample.py": 0.62, + "tests/sync_integration/test_object_manager_batches_patches_edges.py": 6.08, + "tests/sync_integration/test_object_manager_loading.py": 1.97, + "tests/sync_integration/test_object_manager_saving.py": 7.11, + "tests/sync_integration/test_persistence_manager.py": 0.36, + "tests/sync_integration/test_query_dsl.py": 1.55, + "tests/sync_integration/test_recursive_cascade.py": 3.75, + "tests/sync_integration/test_schema.py": 4.26, + "tests/sync_integration/test_schema_runtime.py": 6.71, + "tests/sync_integration/test_search.py": 6.2, + "tests/sync_integration/test_transactions.py": 0.12, + "tests/sync_integration/test_vector_scores.py": 0.51, + "tests/unit/test_assembly_lifecycle.py": 0.95, + "tests/unit/test_bm25.py": 0.0, + "tests/unit/test_cascade.py": 0.01, + "tests/unit/test_case_insensitive.py": 0.02, + "tests/unit/test_ci_trust.py": 1.09, + "tests/unit/test_collection_fields.py": 0.01, "tests/unit/test_compiled_write_kept_rows.py": 0.1, - "tests/unit/test_configuration.py": 0.38, - "tests/unit/test_conflict_sample.py": 0.1, - "tests/unit/test_constraint_manager.py": 0.44, - "tests/unit/test_contention.py": 0.33, - "tests/unit/test_cypher_bc_years.py": 0.41, - "tests/unit/test_cypher_compiled_size.py": 9.34, - "tests/unit/test_cypher_grouped_labels.py": 0.25, - "tests/unit/test_cypher_kind_order.py": 0.05, - "tests/unit/test_cypher_node_equality.py": 1.14, - "tests/unit/test_cypher_parameters.py": 0.87, + "tests/unit/test_configuration.py": 0.01, + "tests/unit/test_conflict_sample.py": 0.0, + "tests/unit/test_constraint_manager.py": 0.77, + "tests/unit/test_contention.py": 0.11, + "tests/unit/test_cypher_bc_years.py": 0.01, + "tests/unit/test_cypher_compiled_size.py": 8.62, + "tests/unit/test_cypher_grouped_labels.py": 0.08, + "tests/unit/test_cypher_kind_order.py": 0.02, + "tests/unit/test_cypher_node_equality.py": 0.39, + "tests/unit/test_cypher_parameters.py": 0.12, "tests/unit/test_cypher_property_key_index.py": 0.1, - "tests/unit/test_cypher_reader.py": 0.79, - "tests/unit/test_cypher_refusals.py": 7.39, - "tests/unit/test_cypher_set_based_delete.py": 0.13, - "tests/unit/test_cypher_shorthand.py": 0.11, - "tests/unit/test_cypher_spec_loader.py": 0.05, - "tests/unit/test_cypher_undirected_join.py": 1.15, - "tests/unit/test_cypher_variables_read_in.py": 0.05, - "tests/unit/test_cypher_zoned_list_sort.py": 0.16, - "tests/unit/test_drivine_test_containers.py": 0.07, - "tests/unit/test_full_text_index_bound.py": 0.12, - "tests/unit/test_golden_statements.py": 0.01, - "tests/unit/test_graph_tables.py": 0.01, - "tests/unit/test_graph_tables_providers.py": 0.03, + "tests/unit/test_cypher_reader.py": 0.67, + "tests/unit/test_cypher_refusals.py": 7.3, + "tests/unit/test_cypher_set_based_delete.py": 0.11, + "tests/unit/test_cypher_shorthand.py": 0.0, + "tests/unit/test_cypher_spec_loader.py": 0.0, + "tests/unit/test_cypher_undirected_join.py": 0.08, + "tests/unit/test_cypher_variables_read_in.py": 0.06, + "tests/unit/test_cypher_zoned_list_sort.py": 1.31, + "tests/unit/test_drivine_test_containers.py": 0.0, + "tests/unit/test_full_text_index_bound.py": 0.02, + "tests/unit/test_golden_statements.py": 0.03, + "tests/unit/test_graph_tables.py": 0.0, + "tests/unit/test_graph_tables_providers.py": 0.02, "tests/unit/test_graph_tables_providers_sync.py": 0.01, - "tests/unit/test_graph_tables_sync.py": 0.02, - "tests/unit/test_model_file.py": 0.37, - "tests/unit/test_model_registry.py": 0.26, - "tests/unit/test_model_registry_cycles.py": 0.08, - "tests/unit/test_object_names.py": 0.02, - "tests/unit/test_optional_antlr.py": 0.03, - "tests/unit/test_oracle_adaptive_plans.py": 0.06, - "tests/unit/test_oracle_compare_text.py": 0.14, - "tests/unit/test_oracle_contains_text.py": 0.05, - "tests/unit/test_oracle_era_functions.py": 0.39, - "tests/unit/test_oracle_graph.py": 0.11, - "tests/unit/test_oracle_item_property.py": 0.4, - "tests/unit/test_oracle_json_table_transform.py": 0.09, - "tests/unit/test_oracle_one_row_plan.py": 0.16, - "tests/unit/test_oracle_optional_join.py": 0.03, - "tests/unit/test_oracle_query_renderer.py": 0.05, - "tests/unit/test_oracle_reverse_text.py": 0.01, - "tests/unit/test_oracle_schema_grammar.py": 0.04, - "tests/unit/test_oracle_search.py": 0.01, - "tests/unit/test_oracle_statement_cache.py": 0.01, - "tests/unit/test_oracle_stored_functions.py": 0.06, - "tests/unit/test_oracle_version.py": 0.03, - "tests/unit/test_postgres_backend.py": 0.02, - "tests/unit/test_postgres_key_probe.py": 0.04, - "tests/unit/test_postgres_nested_loop_plan.py": 0.07, - "tests/unit/test_postgres_spec_compiler.py": 0.04, + "tests/unit/test_graph_tables_sync.py": 0.05, + "tests/unit/test_model_file.py": 0.14, + "tests/unit/test_model_registry.py": 0.01, + "tests/unit/test_model_registry_cycles.py": 0.0, + "tests/unit/test_object_names.py": 0.0, + "tests/unit/test_optional_antlr.py": 0.41, + "tests/unit/test_oracle_adaptive_plans.py": 0.0, + "tests/unit/test_oracle_compare_text.py": 0.19, + "tests/unit/test_oracle_contains_text.py": 0.02, + "tests/unit/test_oracle_era_functions.py": 0.2, + "tests/unit/test_oracle_graph.py": 0.0, + "tests/unit/test_oracle_item_property.py": 0.61, + "tests/unit/test_oracle_json_table_transform.py": 0.04, + "tests/unit/test_oracle_one_row_plan.py": 0.11, + "tests/unit/test_oracle_optional_join.py": 0.12, + "tests/unit/test_oracle_query_renderer.py": 0.0, + "tests/unit/test_oracle_reverse_text.py": 0.0, + "tests/unit/test_oracle_schema_grammar.py": 0.06, + "tests/unit/test_oracle_search.py": 0.0, + "tests/unit/test_oracle_statement_cache.py": 0.0, + "tests/unit/test_oracle_stored_functions.py": 0.03, + "tests/unit/test_oracle_version.py": 0.01, + "tests/unit/test_postgres_backend.py": 0.0, + "tests/unit/test_postgres_key_probe.py": 0.03, + "tests/unit/test_postgres_nested_loop_plan.py": 0.2, + "tests/unit/test_postgres_non_finite.py": 0.04, + "tests/unit/test_postgres_spec_compiler.py": 0.0, "tests/unit/test_query_renderer.py": 0.0, - "tests/unit/test_recorded_inventory.py": 0.01, - "tests/unit/test_release_version.py": 0.07, - "tests/unit/test_save_all_lock_order.py": 0.07, - "tests/unit/test_save_statement_builder.py": 0.34, - "tests/unit/test_schema_commands.py": 0.07, - "tests/unit/test_schema_ddl.py": 0.39, - "tests/unit/test_schema_ddl_spec.py": 0.3, - "tests/unit/test_schema_nothing_to_apply.py": 0.16, - "tests/unit/test_sqlite_cypher_compiler.py": 0.27, - "tests/unit/test_sqlite_cypher_functions.py": 0.39, - "tests/unit/test_sqlite_store.py": 4.88, - "tests/unit/test_store_identity_race.py": 13.18, - "tests/unit/test_stubs.py": 1.5, - "tests/unit/test_sync_up_to_date.py": 0.3, - "tests/unit/test_text_query.py": 5.31, - "tests/unit/test_text_query_counting.py": 1.84, - "tests/unit/test_transactions.py": 1.35, - "tests/unit/test_unasync.py": 0.18, - "tests/unit/test_vector_annotation_check.py": 0.99, - "tests/unit/test_vector_quantization_read.py": 0.63 + "tests/unit/test_recorded_inventory.py": 0.0, + "tests/unit/test_release_version.py": 0.04, + "tests/unit/test_save_all_lock_order.py": 0.01, + "tests/unit/test_save_statement_builder.py": 1.1, + "tests/unit/test_schema_commands.py": 0.0, + "tests/unit/test_schema_ddl.py": 0.01, + "tests/unit/test_schema_ddl_spec.py": 0.01, + "tests/unit/test_schema_nothing_to_apply.py": 0.01, + "tests/unit/test_shards.py": 0.0, + "tests/unit/test_sqlite_cypher_compiler.py": 0.06, + "tests/unit/test_sqlite_cypher_functions.py": 0.29, + "tests/unit/test_sqlite_store.py": 0.03, + "tests/unit/test_store_identity_race.py": 16.28, + "tests/unit/test_stubs.py": 1.29, + "tests/unit/test_sync_up_to_date.py": 3.31, + "tests/unit/test_text_query.py": 2.98, + "tests/unit/test_text_query_counting.py": 0.52, + "tests/unit/test_transactions.py": 0.09, + "tests/unit/test_unasync.py": 0.0, + "tests/unit/test_vector_annotation_check.py": 0.67, + "tests/unit/test_vector_quantization_read.py": 0.05 }, "oracle": { - "tests/integration/oracle/test_oracle_adaptive_plans.py": 8.05, - "tests/integration/oracle/test_oracle_connection.py": 29.52, - "tests/integration/oracle/test_oracle_era_functions.py": 13.42, - "tests/integration/oracle/test_oracle_era_parse.py": 8.05, - "tests/integration/oracle/test_oracle_graph_name.py": 5.37, - "tests/integration/oracle/test_oracle_indexed_filters.py": 13.42, - "tests/integration/oracle/test_oracle_keyed_label_plan.py": 26.83, - "tests/integration/oracle/test_oracle_label_text.py": 16.1, - "tests/integration/oracle/test_oracle_long_text_memory.py": 2.68, - "tests/integration/oracle/test_oracle_node_equality.py": 18.78, - "tests/integration/oracle/test_oracle_nulls_equal.py": 21.47, - "tests/integration/oracle/test_oracle_schema.py": 21.47, - "tests/integration/oracle/test_oracle_schema_catalog.py": 14.89, - "tests/integration/oracle/test_oracle_temporal_without_era.py": 100.3, - "tests/integration/oracle/test_oracle_zoned_list_sort.py": 9.12, - "tests/integration/sql/test_compiled_write_loop_time.py": 11.4, - "tests/integration/sql/test_graph_tables_on_first_use.py": 6.84, - "tests/integration/sql/test_id_key_plans.py": 27.36, - "tests/integration/sql/test_property_key_index.py": 15.78, - "tests/integration/sql/test_schema_ddl.py": 16.87, - "tests/integration/sql/test_sql_fragments.py": 5.91, - "tests/integration/sql/test_sql_saves.py": 12.66, - "tests/integration/sql/test_sql_views.py": 8.44, - "tests/integration/sql/test_sql_writes.py": 5.94, - "tests/integration/test_bm25.py": 5.14, - "tests/integration/test_case_insensitive.py": 4.17, - "tests/integration/test_cypher_compiled.py": 143.46, - "tests/integration/test_cypher_set_based_delete.py": 4.91, - "tests/integration/test_cypher_spec.py": 107.05, - "tests/integration/test_cypher_write_loop.py": 3.29, - "tests/integration/test_grouped_reads.py": 1.99, - "tests/integration/test_index_advice.py": 1.33, - "tests/integration/test_landing_page_sample.py": 0.33, - "tests/integration/test_object_manager_batches_patches_edges.py": 5.98, - "tests/integration/test_object_manager_loading.py": 6.31, - "tests/integration/test_object_manager_saving.py": 12.06, - "tests/integration/test_persistence_manager.py": 1.04, - "tests/integration/test_query_dsl.py": 4.41, - "tests/integration/test_recursive_cascade.py": 7.25, - "tests/integration/test_schema.py": 9.67, - "tests/integration/test_schema_runtime.py": 25.75, - "tests/integration/test_search.py": 66.69, - "tests/integration/test_transactions.py": 14.82, - "tests/integration/test_vector_scores.py": 9.88, - "tests/sync/test_graph_tables_threads.py": 2.47, - "tests/sync/test_store_identity_race.py": 2.47, - "tests/sync/test_threads.py": 4.94, - "tests/sync_integration/oracle/test_oracle_adaptive_plans.py": 7.41, - "tests/sync_integration/oracle/test_oracle_connection.py": 24.7, - "tests/sync_integration/oracle/test_oracle_era_functions.py": 12.35, - "tests/sync_integration/oracle/test_oracle_era_parse.py": 7.41, - "tests/sync_integration/oracle/test_oracle_graph_name.py": 4.94, - "tests/sync_integration/oracle/test_oracle_indexed_filters.py": 7.04, - "tests/sync_integration/oracle/test_oracle_keyed_label_plan.py": 11.43, - "tests/sync_integration/oracle/test_oracle_label_text.py": 6.86, - "tests/sync_integration/oracle/test_oracle_long_text_memory.py": 1.14, - "tests/sync_integration/oracle/test_oracle_node_equality.py": 8.0, - "tests/sync_integration/oracle/test_oracle_nulls_equal.py": 8.0, - "tests/sync_integration/oracle/test_oracle_schema.py": 8.0, - "tests/sync_integration/oracle/test_oracle_schema_catalog.py": 6.86, - "tests/sync_integration/oracle/test_oracle_temporal_without_era.py": 83.02, - "tests/sync_integration/oracle/test_oracle_zoned_list_sort.py": 11.12, - "tests/sync_integration/sql/test_compiled_write_loop_time.py": 13.9, - "tests/sync_integration/sql/test_graph_tables_on_first_use.py": 2.78, - "tests/sync_integration/sql/test_id_key_plans.py": 33.36, - "tests/sync_integration/sql/test_property_key_index.py": 47.26, - "tests/sync_integration/sql/test_schema_ddl.py": 37.98, - "tests/sync_integration/sql/test_sql_fragments.py": 2.14, - "tests/sync_integration/sql/test_sql_saves.py": 4.29, - "tests/sync_integration/sql/test_sql_views.py": 3.06, - "tests/sync_integration/sql/test_sql_writes.py": 3.67, - "tests/sync_integration/test_bm25.py": 4.9, - "tests/sync_integration/test_case_insensitive.py": 5.81, - "tests/sync_integration/test_cypher_compiled.py": 125.86, - "tests/sync_integration/test_cypher_set_based_delete.py": 4.79, - "tests/sync_integration/test_cypher_spec.py": 98.57, - "tests/sync_integration/test_cypher_write_loop.py": 4.95, - "tests/sync_integration/test_grouped_reads.py": 7.42, - "tests/sync_integration/test_index_advice.py": 4.95, - "tests/sync_integration/test_landing_page_sample.py": 1.24, - "tests/sync_integration/test_object_manager_batches_patches_edges.py": 22.27, - "tests/sync_integration/test_object_manager_loading.py": 2.96, - "tests/sync_integration/test_object_manager_saving.py": 3.75, - "tests/sync_integration/test_persistence_manager.py": 1.63, - "tests/sync_integration/test_query_dsl.py": 2.44, - "tests/sync_integration/test_recursive_cascade.py": 2.44, - "tests/sync_integration/test_schema.py": 4.38, - "tests/sync_integration/test_schema_runtime.py": 6.16, - "tests/sync_integration/test_search.py": 5.94, - "tests/sync_integration/test_transactions.py": 0.64, - "tests/sync_integration/test_vector_scores.py": 0.39, - "tests/unit/test_bm25.py": 0.39, - "tests/unit/test_cascade.py": 0.59, - "tests/unit/test_case_insensitive.py": 0.29, - "tests/unit/test_compiled_write_kept_rows.py": 0.2, - "tests/unit/test_conflict_sample.py": 0.2, - "tests/unit/test_contention.py": 0.29, - "tests/unit/test_cypher_bc_years.py": 0.2, - "tests/unit/test_cypher_compiled_size.py": 4.54, - "tests/unit/test_cypher_kind_order.py": 0.03, - "tests/unit/test_cypher_node_equality.py": 0.69, - "tests/unit/test_cypher_parameters.py": 0.23, - "tests/unit/test_cypher_property_key_index.py": 0.12, - "tests/unit/test_cypher_refusals.py": 3.56, - "tests/unit/test_cypher_set_based_delete.py": 0.3, - "tests/unit/test_cypher_undirected_join.py": 0.25, - "tests/unit/test_cypher_zoned_list_sort.py": 0.09, - "tests/unit/test_drivine_test_containers.py": 0.01, - "tests/unit/test_full_text_index_bound.py": 0.12, + "tests/integration/oracle/test_oracle_connection.py": 6.78, + "tests/integration/oracle/test_oracle_era_functions.py": 3.25, + "tests/integration/oracle/test_oracle_graph_name.py": 1.57, + "tests/integration/oracle/test_oracle_indexed_filters.py": 4.16, + "tests/integration/oracle/test_oracle_label_text.py": 0.95, + "tests/integration/oracle/test_oracle_nulls_equal.py": 6.12, + "tests/integration/oracle/test_oracle_schema.py": 22.43, + "tests/integration/oracle/test_oracle_schema_catalog.py": 10.27, + "tests/integration/oracle/test_oracle_temporal_without_era.py": 46.71, + "tests/integration/oracle/test_oracle_zoned_list_sort.py": 7.57, + "tests/integration/sql/test_graph_tables_on_first_use.py": 2.04, + "tests/integration/sql/test_id_key_plans.py": 7.2, + "tests/integration/sql/test_property_key_index.py": 20.4, + "tests/integration/sql/test_schema_ddl.py": 26.59, + "tests/integration/sql/test_sql_fragments.py": 0.18, + "tests/integration/sql/test_sql_saves.py": 1.41, + "tests/integration/sql/test_sql_views.py": 0.51, + "tests/integration/sql/test_sql_writes.py": 0.85, + "tests/integration/test_bm25.py": 11.63, + "tests/integration/test_case_insensitive.py": 1.81, + "tests/integration/test_cypher_compiled.py": 117.53, + "tests/integration/test_cypher_set_based_delete.py": 1.16, + "tests/integration/test_cypher_spec.py": 92.55, + "tests/integration/test_cypher_write_loop.py": 0.93, + "tests/integration/test_grouped_reads.py": 0.23, + "tests/integration/test_index_advice.py": 0.56, + "tests/integration/test_landing_page_sample.py": 0.12, + "tests/integration/test_object_manager_batches_patches_edges.py": 14.67, + "tests/integration/test_object_manager_loading.py": 0.82, + "tests/integration/test_object_manager_saving.py": 5.43, + "tests/integration/test_persistence_manager.py": 0.24, + "tests/integration/test_query_dsl.py": 0.55, + "tests/integration/test_recursive_cascade.py": 1.73, + "tests/integration/test_schema.py": 25.2, + "tests/integration/test_schema_runtime.py": 8.23, + "tests/integration/test_search.py": 5.65, + "tests/integration/test_transactions.py": 0.12, + "tests/integration/test_vector_scores.py": 0.9, + "tests/sync/test_graph_tables_threads.py": 0.39, + "tests/sync/test_store_identity_race.py": 28.07, + "tests/sync/test_threads.py": 0.2, + "tests/sync_integration/oracle/test_oracle_connection.py": 3.86, + "tests/sync_integration/oracle/test_oracle_era_functions.py": 1.87, + "tests/sync_integration/oracle/test_oracle_graph_name.py": 0.7, + "tests/sync_integration/oracle/test_oracle_indexed_filters.py": 4.03, + "tests/sync_integration/oracle/test_oracle_label_text.py": 0.23, + "tests/sync_integration/oracle/test_oracle_nulls_equal.py": 3.51, + "tests/sync_integration/oracle/test_oracle_schema.py": 4.05, + "tests/sync_integration/oracle/test_oracle_schema_catalog.py": 10.41, + "tests/sync_integration/oracle/test_oracle_temporal_without_era.py": 43.34, + "tests/sync_integration/oracle/test_oracle_zoned_list_sort.py": 7.6, + "tests/sync_integration/sql/test_graph_tables_on_first_use.py": 0.53, + "tests/sync_integration/sql/test_id_key_plans.py": 5.78, + "tests/sync_integration/sql/test_property_key_index.py": 19.61, + "tests/sync_integration/sql/test_schema_ddl.py": 26.22, + "tests/sync_integration/sql/test_sql_fragments.py": 0.48, + "tests/sync_integration/sql/test_sql_saves.py": 1.51, + "tests/sync_integration/sql/test_sql_views.py": 0.51, + "tests/sync_integration/sql/test_sql_writes.py": 0.76, + "tests/sync_integration/test_bm25.py": 9.22, + "tests/sync_integration/test_case_insensitive.py": 1.79, + "tests/sync_integration/test_cypher_compiled.py": 114.13, + "tests/sync_integration/test_cypher_set_based_delete.py": 0.96, + "tests/sync_integration/test_cypher_spec.py": 101.14, + "tests/sync_integration/test_cypher_write_loop.py": 0.98, + "tests/sync_integration/test_grouped_reads.py": 0.23, + "tests/sync_integration/test_index_advice.py": 0.32, + "tests/sync_integration/test_landing_page_sample.py": 0.22, + "tests/sync_integration/test_object_manager_batches_patches_edges.py": 16.97, + "tests/sync_integration/test_object_manager_loading.py": 1.1, + "tests/sync_integration/test_object_manager_saving.py": 0.95, + "tests/sync_integration/test_persistence_manager.py": 0.24, + "tests/sync_integration/test_query_dsl.py": 0.74, + "tests/sync_integration/test_recursive_cascade.py": 1.44, + "tests/sync_integration/test_schema.py": 5.65, + "tests/sync_integration/test_schema_runtime.py": 9.63, + "tests/sync_integration/test_search.py": 3.94, + "tests/sync_integration/test_transactions.py": 0.08, + "tests/sync_integration/test_vector_scores.py": 0.59, + "tests/unit/test_bm25.py": 0.0, + "tests/unit/test_cascade.py": 0.0, + "tests/unit/test_case_insensitive.py": 0.0, + "tests/unit/test_compiled_write_kept_rows.py": 0.08, + "tests/unit/test_conflict_sample.py": 0.0, + "tests/unit/test_contention.py": 0.02, + "tests/unit/test_cypher_bc_years.py": 0.0, + "tests/unit/test_cypher_compiled_size.py": 5.44, + "tests/unit/test_cypher_kind_order.py": 0.02, + "tests/unit/test_cypher_node_equality.py": 0.25, + "tests/unit/test_cypher_parameters.py": 0.06, + "tests/unit/test_cypher_property_key_index.py": 0.04, + "tests/unit/test_cypher_refusals.py": 2.83, + "tests/unit/test_cypher_set_based_delete.py": 0.06, + "tests/unit/test_cypher_undirected_join.py": 0.04, + "tests/unit/test_cypher_zoned_list_sort.py": 1.15, + "tests/unit/test_drivine_test_containers.py": 0.0, + "tests/unit/test_full_text_index_bound.py": 0.0, "tests/unit/test_golden_statements.py": 0.01, - "tests/unit/test_graph_tables_providers.py": 0.13, - "tests/unit/test_graph_tables_providers_sync.py": 0.06, - "tests/unit/test_oracle_adaptive_plans.py": 0.06, - "tests/unit/test_oracle_compare_text.py": 0.1, - "tests/unit/test_oracle_contains_text.py": 0.03, - "tests/unit/test_oracle_era_functions.py": 0.25, - "tests/unit/test_oracle_graph.py": 0.07, - "tests/unit/test_oracle_item_property.py": 0.25, - "tests/unit/test_oracle_json_table_transform.py": 0.06, - "tests/unit/test_oracle_one_row_plan.py": 0.7, - "tests/unit/test_oracle_optional_join.py": 0.14, - "tests/unit/test_oracle_query_renderer.py": 0.28, - "tests/unit/test_oracle_reverse_text.py": 0.14, - "tests/unit/test_oracle_schema_grammar.py": 0.56, - "tests/unit/test_oracle_search.py": 0.14, - "tests/unit/test_oracle_statement_cache.py": 0.14, - "tests/unit/test_oracle_stored_functions.py": 0.98, - "tests/unit/test_oracle_version.py": 0.42, - "tests/unit/test_postgres_key_probe.py": 0.07, - "tests/unit/test_postgres_nested_loop_plan.py": 0.07, - "tests/unit/test_save_all_lock_order.py": 0.07, - "tests/unit/test_schema_ddl.py": 0.07, - "tests/unit/test_sqlite_cypher_compiler.py": 0.07, - "tests/unit/test_sqlite_store.py": 0.07, - "tests/unit/test_store_identity_race.py": 0.56, - "tests/unit/test_text_query.py": 3.62, - "tests/unit/test_text_query_counting.py": 1.02 + "tests/unit/test_graph_tables_providers.py": 0.01, + "tests/unit/test_graph_tables_providers_sync.py": 0.39, + "tests/unit/test_oracle_adaptive_plans.py": 0.0, + "tests/unit/test_oracle_compare_text.py": 0.17, + "tests/unit/test_oracle_contains_text.py": 0.01, + "tests/unit/test_oracle_era_functions.py": 0.09, + "tests/unit/test_oracle_graph.py": 0.0, + "tests/unit/test_oracle_item_property.py": 0.29, + "tests/unit/test_oracle_json_table_transform.py": 0.03, + "tests/unit/test_oracle_one_row_plan.py": 0.07, + "tests/unit/test_oracle_optional_join.py": 0.06, + "tests/unit/test_oracle_query_renderer.py": 0.0, + "tests/unit/test_oracle_reverse_text.py": 0.0, + "tests/unit/test_oracle_schema_grammar.py": 0.06, + "tests/unit/test_oracle_search.py": 0.0, + "tests/unit/test_oracle_statement_cache.py": 0.0, + "tests/unit/test_oracle_stored_functions.py": 0.03, + "tests/unit/test_oracle_version.py": 0.0, + "tests/unit/test_postgres_key_probe.py": 0.0, + "tests/unit/test_postgres_nested_loop_plan.py": 0.0, + "tests/unit/test_postgres_non_finite.py": 0.01, + "tests/unit/test_save_all_lock_order.py": 0.0, + "tests/unit/test_schema_ddl.py": 0.0, + "tests/unit/test_sqlite_cypher_compiler.py": 0.02, + "tests/unit/test_sqlite_store.py": 0.0, + "tests/unit/test_store_identity_race.py": 4.89, + "tests/unit/test_text_query.py": 0.02, + "tests/unit/test_text_query_counting.py": 0.25 } } From f187232201ee0fd1f48c64e689a4ef19dd0027d3 Mon Sep 17 00:00:00 2001 From: jasper blues Date: Sun, 8 Nov 2026 14:59:11 +1000 Subject: [PATCH 3/5] Count the write loop's self-loops from their one node, as two node scans took 8 minutes on Postgres The MERGE check joined the node table twice, and with the statistics the suite leaves on purpose (#430) Postgres planned it as a nested loop over every node for each node (liberation-data/drivine#680). --- tests/integration/sql/test_compiled_write_loop_time.py | 5 ++++- tests/sync_integration/sql/test_compiled_write_loop_time.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/integration/sql/test_compiled_write_loop_time.py b/tests/integration/sql/test_compiled_write_loop_time.py index df29012c..7f0e4c33 100644 --- a/tests/integration/sql/test_compiled_write_loop_time.py +++ b/tests/integration/sql/test_compiled_write_loop_time.py @@ -59,10 +59,13 @@ class Write: Write("CREATE", "UNWIND range(1, $n) AS i CREATE (:CyLoop {i: i})", "MATCH (n:CyLoop) RETURN count(n)", None), Write("SET", "MATCH (n:CyLoop) SET n.x = n.i", "MATCH (n:CyLoop) WHERE n.x IS NOT NULL RETURN count(n)", None), Write("REMOVE", "MATCH (n:CyLoop) REMOVE n.x", "MATCH (n:CyLoop) WHERE n.x IS NOT NULL RETURN count(n)", 0), + # The check reads each self-loop from its one node. Read as `(:CyLoop)-[r:CY_SELF]->(:CyLoop)`, it joined the node + # table twice, which on Postgres, its statistics left saying the table is empty (#430), was planned as a nested + # loop over every node for each node: 8 minutes over 20,000 rows, untimed, in CI (liberation-data/drivine#680). Write( "MERGE", "MATCH (n:CyLoop) MERGE (n)-[:CY_SELF]->(n)", - "MATCH (:CyLoop)-[r:CY_SELF]->(:CyLoop) RETURN count(r)", + "MATCH (n:CyLoop)-[r:CY_SELF]->(n) RETURN count(r)", None, ), Write( diff --git a/tests/sync_integration/sql/test_compiled_write_loop_time.py b/tests/sync_integration/sql/test_compiled_write_loop_time.py index 691e47d4..1d975bf1 100644 --- a/tests/sync_integration/sql/test_compiled_write_loop_time.py +++ b/tests/sync_integration/sql/test_compiled_write_loop_time.py @@ -61,10 +61,13 @@ class Write: Write("CREATE", "UNWIND range(1, $n) AS i CREATE (:CyLoop {i: i})", "MATCH (n:CyLoop) RETURN count(n)", None), Write("SET", "MATCH (n:CyLoop) SET n.x = n.i", "MATCH (n:CyLoop) WHERE n.x IS NOT NULL RETURN count(n)", None), Write("REMOVE", "MATCH (n:CyLoop) REMOVE n.x", "MATCH (n:CyLoop) WHERE n.x IS NOT NULL RETURN count(n)", 0), + # The check reads each self-loop from its one node. Read as `(:CyLoop)-[r:CY_SELF]->(:CyLoop)`, it joined the node + # table twice, which on Postgres, its statistics left saying the table is empty (#430), was planned as a nested + # loop over every node for each node: 8 minutes over 20,000 rows, untimed, in CI (liberation-data/drivine#680). Write( "MERGE", "MATCH (n:CyLoop) MERGE (n)-[:CY_SELF]->(n)", - "MATCH (:CyLoop)-[r:CY_SELF]->(:CyLoop) RETURN count(r)", + "MATCH (n:CyLoop)-[r:CY_SELF]->(n) RETURN count(r)", None, ), Write( From f91d51622969ead442a09cd4d290036a8a93dfe0 Mon Sep 17 00:00:00 2001 From: jasper blues Date: Sun, 8 Nov 2026 17:00:01 +1000 Subject: [PATCH 4/5] Run the database suite on Python 3.11 and 3.14 alone on a pull request, and tests/unit on 3.12 and 3.13 --- .github/workflows/ci.yml | 27 ++++++++++++++++----------- docs/11-testing.md | 5 +++-- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5751e658..bfcc2f50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,8 @@ name: CI # Lints and tests drivine on each Python it supports, and again with Oracle, each run split over parallel jobs with # containers of their own: `pytest --shard i/n` runs one part, a module whole in one part (tests/shards.py). # -# A pull request's run leaves out the nightly tests (`-m nightly`, the tests marked so): slow ones, and ones that rest +# A pull request's run takes the database suite on Python 3.11 and 3.14 alone, and tests/unit on 3.12 and 3.13; a full +# run takes it on all four. A pull request's run also leaves out the nightly tests (`-m nightly`, the tests marked so): slow ones, and ones that rest # on a timing or a plan. A full run has them too, in a part of their own: a push to main, a pull request labelled # `full-ci`, nightly.yml each night, and release.yml's gate. release.yml calls this as the gate before a release, # unless a full push or pull_request run of it has already passed on the same tree. @@ -73,9 +74,10 @@ jobs: - name: Check the Cypher spec run: python scripts/check_cypher_spec.py - # Every test but Oracle's, on each Python, in three parts, and the nightly tests in a fourth on a full run (picked by - # the same test as `tree`'s record). The first part lints too. The parts of a pull request's run hold every test but - # the nightly ones once. + # Every test but Oracle's, in three parts, and the nightly tests in a fourth: on each Python on a full run (picked by + # the same test as `tree`'s record). A pull request's run takes the database suite on the oldest and the newest + # Python alone, 3.11 and 3.14, in the three parts, which hold every test but the nightly ones once; on 3.12 and 3.13 + # it runs tests/unit alone, which needs no database, with no containers. The first part, or the unit part, lints too. test: name: test (${{ matrix.python }}, ${{ matrix.part }}) runs-on: ubuntu-latest @@ -83,27 +85,30 @@ jobs: strategy: fail-fast: false matrix: - python: ["3.11", "3.12", "3.13", "3.14"] - part: ${{ fromJSON((inputs.full || github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'full-ci')) && '["1/3", "2/3", "3/3", "nightly"]' || '["1/3", "2/3", "3/3"]') }} + include: ${{ fromJSON((inputs.full || github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'full-ci')) && '[{"python":"3.11","part":"1/3"},{"python":"3.11","part":"2/3"},{"python":"3.11","part":"3/3"},{"python":"3.11","part":"nightly"},{"python":"3.12","part":"1/3"},{"python":"3.12","part":"2/3"},{"python":"3.12","part":"3/3"},{"python":"3.12","part":"nightly"},{"python":"3.13","part":"1/3"},{"python":"3.13","part":"2/3"},{"python":"3.13","part":"3/3"},{"python":"3.13","part":"nightly"},{"python":"3.14","part":"1/3"},{"python":"3.14","part":"2/3"},{"python":"3.14","part":"3/3"},{"python":"3.14","part":"nightly"}]' || '[{"python":"3.11","part":"1/3"},{"python":"3.11","part":"2/3"},{"python":"3.11","part":"3/3"},{"python":"3.14","part":"1/3"},{"python":"3.14","part":"2/3"},{"python":"3.14","part":"3/3"},{"python":"3.12","part":"unit"},{"python":"3.13","part":"unit"}]') }} steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 with: python-version: ${{ matrix.python }} - run: uv sync - - if: matrix.part == '1/3' + - if: matrix.part == '1/3' || matrix.part == 'unit' run: uv run ruff check . - - if: matrix.part == '1/3' + - if: matrix.part == '1/3' || matrix.part == 'unit' run: uv run ruff format --check . - - if: matrix.part == '1/3' + - if: matrix.part == '1/3' || matrix.part == 'unit' run: uv run pyright - - if: matrix.part == '1/3' + - if: matrix.part == '1/3' || matrix.part == 'unit' run: uv run python scripts/unasync.py --check - name: Run the tests env: PART: ${{ matrix.part }} run: | - if [ "$PART" = nightly ]; then picked=(-m nightly); else picked=(-m "not nightly" --shard "$PART"); fi + case "$PART" in + nightly) picked=(-m nightly) ;; + unit) picked=(-o drivine_test_containers=false tests/unit) ;; + *) picked=(-m "not nightly" --shard "$PART") ;; + esac uv run pytest -q --durations=15 --record-durations durations.json "${picked[@]}" # How long each module took, to balance the parts by (tests/durations.json). From the slowest Python alone. - if: always() && matrix.python == '3.11' diff --git a/docs/11-testing.md b/docs/11-testing.md index 7deccd8c..108d9ab8 100644 --- a/docs/11-testing.md +++ b/docs/11-testing.md @@ -18,8 +18,9 @@ release reaches your tests as soon as it is out; pin one with its `_TEST_I Drivine's own tests run on Oracle only with `ORACLE_TESTS=true`: its image is large, and the database wants two gigabytes. A run without it ends with the line `Oracle tests skipped: set ORACLE_TESTS=true to run them`. CI runs the Oracle tests in jobs of their own, with `pytest -k oracle`. CI splits each run over parallel jobs, `pytest --shard i/n` -(tests/shards.py), and a pull request's run leaves out the tests marked `nightly`: the timings, Oracle's plans, and the -texts past 32 KB. They run each night (`.github/workflows/nightly.yml`, which opens an issue labelled `nightly` when +(tests/shards.py), a pull request's run takes the database suite on Python 3.11 and 3.14 alone (3.12 and 3.13 run `tests/unit`), and it +leaves out the tests marked `nightly`: the timings, Oracle's plans, and the +texts past 32 KB. Those, and the database suite on every Python, run each night (`.github/workflows/nightly.yml`, which opens an issue labelled `nightly` when they fail), on each push to main, in a release's gate, and on a pull request labelled `full-ci`. A local `pytest` runs them all. The tests run on the plugin's own images, the latest Neo4j, Memgraph and FalkorDB, so that an engine release that changes what Drivine reads or writes shows up at once. CI holds From 63cca4d1019ecb360a74392fe8c9b225115175bc Mon Sep 17 00:00:00 2001 From: jasper blues Date: Sun, 8 Nov 2026 17:28:51 +1000 Subject: [PATCH 5/5] Report a hanging test in a scheduled nightly, keep the nightly issue under GitHub's size limit, and test the shard hooks --- .github/workflows/nightly.yml | 34 +++++++++++++------ scripts/ci_trust.py | 4 ++- tests/unit/test_ci_trust.py | 15 +++++++++ tests/unit/test_shards.py | 61 +++++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 10 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 70b76433..56f95191 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -38,27 +38,37 @@ jobs: env: GH_TOKEN: ${{ github.token }} RESULT: ${{ needs.ci.result }} + EVENT: ${{ github.event_name }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} POST: ${{ github.ref == 'refs/heads/main' }} run: | repo="$GITHUB_REPOSITORY" open=$(gh issue list --repo "$repo" --label nightly --state open --json number -q '.[0].number') - # Writes to the issue on main, and to the summary alone elsewhere. + # Writes to the issue on main, and to the summary alone elsewhere. An issue's body or comment holds 65,536 + # characters at most, and GitHub refuses a longer one, so the issue gets the first 60,000, and the summary + # the whole report. post() { echo "$1" echo "$1" >> "$GITHUB_STEP_SUMMARY" [ "$POST" = true ] || return 0 + local text=$1 + if [ "${#text}" -gt 60000 ]; then + text="${text:0:60000}"$'\n\n'"Cut short, as an issue holds 65,536 characters: the run's summary has the whole report, $RUN_URL" + fi if [ "$2" = close ]; then - gh issue close "$open" --repo "$repo" --comment "$1" + gh issue close "$open" --repo "$repo" --comment "$text" elif [ -n "$open" ]; then - gh issue comment "$open" --repo "$repo" --body "$1" + gh issue comment "$open" --repo "$repo" --body "$text" else gh label create nightly --repo "$repo" --color B60205 --description "The nightly CI failed" 2> /dev/null || true - gh issue create --repo "$repo" --label nightly --title "Nightly CI fails" --body "$1" + gh issue create --repo "$repo" --label nightly --title "Nightly CI fails" --body "$text" fi } - # A run cancelled by hand says nothing of the tree. - [ "$RESULT" != cancelled ] || exit 0 + # A run cancelled by hand says nothing of the tree. A scheduled run is never cancelled by hand, and a job past + # its timeout-minutes can conclude cancelled, so there a cancelled run or job is a hanging test: a failure. + [ "$RESULT" != cancelled ] || [ "$EVENT" = schedule ] || exit 0 + picked='.conclusion == "failure" or .conclusion == "timed_out"' + [ "$EVENT" != schedule ] || picked="$picked or .conclusion == \"cancelled\"" if [ "$RESULT" = success ]; then [ -z "$open" ] || post "The nightly passed again on $GITHUB_SHA: $RUN_URL" close exit 0 @@ -68,9 +78,13 @@ jobs: echo "The nightly on $GITHUB_SHA concluded $RESULT: $RUN_URL" echo gh api --paginate "repos/$repo/actions/runs/$GITHUB_RUN_ID/jobs?per_page=100" \ - -q '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out") | "\(.id)\t\(.name)\t\(.html_url)"' | - while IFS=$'\t' read -r id name url; do - echo "### [$name]($url)" + -q ".jobs[] | select($picked) | \"\\(.id)\\t\\(.name)\\t\\(.conclusion)\\t\\(.html_url)\"" | + while IFS=$'\t' read -r id name conclusion url; do + if [ "$conclusion" = cancelled ]; then + echo "### [$name]($url), cancelled or timed out" + else + echo "### [$name]($url)" + fi echo # pytest's short summary, a line for each failed test and each error, without the log's timestamps. failing=$(gh api --allow-escape-sequences "repos/$repo/actions/jobs/$id/logs" | @@ -78,6 +92,8 @@ jobs: cut -c1-300 | awk '!seen[$0]++' | head -40 || true) if [ -n "$failing" ]; then printf '```\n%s\n```\n' "$failing" + elif [ "$conclusion" = cancelled ]; then + echo "No failing test in its log: it was stopped before pytest's summary, at a test that hangs or past its time." else echo "No failing test in its log: it failed in a step around the tests, or waits for jobs above." fi diff --git a/scripts/ci_trust.py b/scripts/ci_trust.py index 46657fce..595bfbf9 100644 --- a/scripts/ci_trust.py +++ b/scripts/ci_trust.py @@ -165,7 +165,9 @@ def look(state: str, reason: str) -> Look: ) if run.get("event") == "pull_request" and tree not in trees: if trees: - return look("ignored", f"it tested its merge ref, tree {trees[0]}, as its base had moved on") + return look( + "ignored", f"it tested its merge ref, tree {trees[0].removeprefix(PR_SET)}, as its base had moved on" + ) if not done: return look("maybe", "it has not yet recorded the tree of the merge ref it tests") return look( diff --git a/tests/unit/test_ci_trust.py b/tests/unit/test_ci_trust.py index f540c597..85e9a9a9 100644 --- a/tests/unit/test_ci_trust.py +++ b/tests/unit/test_ci_trust.py @@ -194,6 +194,21 @@ def test_doesnt_trust_a_pull_requests_run_that_tested_another_tree_as_its_base_h assert f"it tested its merge ref, tree {OTHER}, as its base had moved on" in result.text +def test_names_the_tree_alone_of_a_pull_requests_run_of_the_pr_set_that_tested_another_tree() -> None: + result = decide( + { + runs_on(SHA): workflow_runs(), + RECORDS: record_on(), + runs_on(HEAD): workflow_runs(run(4, event="pull_request", head_sha=HEAD)), + artifacts_of(4): recorded(f"pr-set-{OTHER}"), + }, + [SHA, HEAD], + ) + + assert result.decision == "gate" + assert f"it tested its merge ref, tree {OTHER}, as its base had moved on" in result.text + + def test_doesnt_believe_a_record_on_a_run_whose_head_commit_is_another_tree_as_a_forks_ci_could_make_it() -> None: forged = run(5, event="pull_request", head_sha=HEAD, tree=OTHER) result = decide({runs_on(SHA): workflow_runs(), RECORDS: record_on(5), run_path(5): forged}) diff --git a/tests/unit/test_shards.py b/tests/unit/test_shards.py index f07ab0d8..c070d8cf 100644 --- a/tests/unit/test_shards.py +++ b/tests/unit/test_shards.py @@ -1,7 +1,13 @@ """tests/shards.py: CI's parts of a run, each module whole in one part, balanced by the time each took.""" +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any + import pytest +from tests import shards from tests.shards import parse, parts @@ -35,3 +41,58 @@ def test_a_part_is_i_of_n_numbered_from_1(text: str) -> None: def test_parses_i_of_n() -> None: assert parse("2/3") == (2, 3) + + +def test_reads_the_oracle_durations_with_oracle_tests_and_the_default_ones_without( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + durations = tmp_path / "durations.json" + durations.write_text(json.dumps({"default": {"a.py": 1.0}, "oracle": {"b.py": 2.0}})) + monkeypatch.setattr(shards, "DURATIONS", durations) + + monkeypatch.delenv("ORACLE_TESTS", raising=False) + assert shards.recorded() == {"a.py": 1.0} + monkeypatch.setenv("ORACLE_TESTS", "true") + assert shards.recorded() == {"b.py": 2.0} + monkeypatch.setattr(shards, "DURATIONS", tmp_path / "none.json") + assert shards.recorded() == {} + + +def collected(*nodeids: str) -> list[Any]: + return [SimpleNamespace(nodeid=it) for it in nodeids] + + +def config(options: dict[str, str | None], deselected: list[Any]) -> Any: + hook = SimpleNamespace(pytest_deselected=lambda items: deselected.extend(items)) + return SimpleNamespace(getoption=lambda name: options.get(name), hook=hook) + + +def test_each_part_keeps_its_modules_whole_and_reports_the_rest_deselected(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(shards, "recorded", lambda: {"a.py": 100.0, "b.py": 10.0}) + picked = collected("a.py::one", "b.py::one", "a.py::two", "b.py::two") + kept: list[list[str]] = [] + for part in ("1/2", "2/2"): + items, deselected = list(picked), [] + shards.pytest_collection_modifyitems(config({"--shard": part}, deselected), items) + kept.append([it.nodeid for it in items]) + assert sorted(it.nodeid for it in items + deselected) == sorted(it.nodeid for it in picked) + + assert kept == [["a.py::one", "a.py::two"], ["b.py::one", "b.py::two"]] + + +def test_without_a_part_every_test_is_kept() -> None: + items, deselected = collected("a.py::one", "b.py::one"), [] + shards.pytest_collection_modifyitems(config({}, deselected), items) + assert len(items) == 2 and deselected == [] + + +def test_records_the_seconds_each_module_took_setup_and_teardown_with_it( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(shards, "TOOK", shards.defaultdict(float)) + for nodeid, duration in [("b.py::one", 1.0), ("a.py::one", 0.25), ("b.py::one", 0.5), ("b.py::two", 2.004)]: + shards.pytest_runtest_logreport(SimpleNamespace(nodeid=nodeid, duration=duration)) + path = tmp_path / "took.json" + shards.pytest_sessionfinish(SimpleNamespace(config=config({"--record-durations": str(path)}, []))) + + assert json.loads(path.read_text()) == {"a.py": 0.25, "b.py": 3.5}