Split CI's tests over parallel jobs, and move the slow suites to a nightly

.github/workflows/ci.yml

@@ -1,31 +1,47 @@
 name: CI
 
+# Lints and tests drivine on each Python it supports, and again with Oracle, each run split over parallel jobs with
+# containers of their own: `pytest --shard i/n` runs one part, a module whole in one part (tests/shards.py).
+#
+# A pull request's run takes the database suite on Python 3.11 and 3.14 alone, and tests/unit on 3.12 and 3.13; a full
+# run takes it on all four. A pull request's run also leaves out the nightly tests (`-m nightly`, the tests marked so): slow ones, and ones that rest
+# on a timing or a plan. A full run has them too, in a part of their own: a push to main, a pull request labelled
+# `full-ci`, nightly.yml each night, and release.yml's gate. release.yml calls this as the gate before a release,
+# unless a full push or pull_request run of it has already passed on the same tree.
 on:
   push:
     branches: [main]
   pull_request:
-  # release.yml runs this as its gate, so that a release passes the same CI as a pull request, unless a push or
-  # pull_request run of it has already passed on the same tree.
+    # As the default, and labeled, so that labelling a pull request `full-ci` runs the full set on it.
+    types: [opened, synchronize, reopened, labeled]
   workflow_call:
-  # Each night on the latest images, without the pin below, so an engine's new release shows up here and not in a pull
-  # request's run.
-  schedule:
-    - cron: "17 4 * * *"
+    inputs:
+      full:
+        description: Run the nightly tests too, as a release and the nightly do.
+        type: boolean
+        default: false
 
-# FalkorDB is held at a release that passes: a pull request's run takes falkordb/falkordb:6.0.3. The nightly run takes
-# ghcr.io/falkordb/falkordb:latest, FalkorDB's newest release, and not the falkordb/falkordb:latest the plugin names:
-# Docker Hub's latest is whichever release FalkorDB pushed last, and that can be an older line's backport, as v4.22.3
-# took it after 6.0.3 (no =~, and no pattern comprehension in a function, FalkorDB/FalkorDB#3551). Drop the nightly's
-# ghcr.io image once Docker Hub's latest follows FalkorDB's newest line again; move the pin, or drop it, once the
-# nightly passes.
+# A pull request's newer run, of a later push or a label, takes the place of the one still going. Any other run, and
+# this workflow called by another, is a group of its own.
+concurrency:
+  group: ${{ github.workflow == 'CI' && github.event_name == 'pull_request' && format('ci-pr-{0}', github.event.pull_request.number) || format('ci-{0}-{1}', github.workflow, github.run_id) }}
+  cancel-in-progress: true
+
+# FalkorDB is held at a release that passes: a pull request's run takes falkordb/falkordb:6.0.3. The nightly run
+# (nightly.yml, whose name this sees as its workflow) takes ghcr.io/falkordb/falkordb:latest, FalkorDB's newest release,
+# and not the falkordb/falkordb:latest the plugin names: Docker Hub's latest is whichever release FalkorDB pushed last,
+# and that can be an older line's backport, as v4.22.3 took it after 6.0.3 (no =~, and no pattern comprehension in a
+# function, FalkorDB/FalkorDB#3551). Drop the nightly's ghcr.io image once Docker Hub's latest follows FalkorDB's newest
+# line again; move the pin, or drop it, once the nightly passes.
 env:
-  FALKORDB_TEST_IMAGE: ${{ github.event_name == 'schedule' && 'ghcr.io/falkordb/falkordb:latest' || 'falkordb/falkordb:6.0.3' }}
+  FALKORDB_TEST_IMAGE: ${{ github.workflow == 'Nightly' && 'ghcr.io/falkordb/falkordb:latest' || 'falkordb/falkordb:6.0.3' }}
 
 jobs:
   # Records the git tree this run tests, as an artifact named ci-tree-<tree>, so a release of the same tree can trust
   # this run and skip its gate (scripts/ci_trust.py). It matters on a pull request, which tests its merge ref, not the
-  # head commit the run names. It has no condition, as no job here may: release.yml trusts a run only when every job
-  # in it passed, and a job skipped by a condition doesn't.
+  # head commit the run names. A run of the pull request set alone, without the nightly tests, names it
+  # ci-tree-pr-set-<tree>, which a release doesn't trust. It has no condition, as no job here may: release.yml trusts
+  # a run only when every job in it passed, and a job skipped by a condition doesn't.
   tree:
     name: record the tree
     runs-on: ubuntu-latest
@@ -41,7 +57,7 @@ jobs:
       # Overwritten when the run is re-run, as an artifact's name is unique within a run.
       - uses: actions/upload-artifact@v4
         with:
-          name: ci-tree-${{ steps.tree.outputs.tree }}
+          name: ci-tree-${{ !(inputs.full || github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'full-ci')) && 'pr-set-' || '' }}${{ steps.tree.outputs.tree }}
           path: ci-tree.txt
           if-no-files-found: error
           overwrite: true
@@ -58,30 +74,61 @@ jobs:
       - name: Check the Cypher spec
         run: python scripts/check_cypher_spec.py
 
+  # Every test but Oracle's, in three parts, and the nightly tests in a fourth: on each Python on a full run (picked by
+  # the same test as `tree`'s record). A pull request's run takes the database suite on the oldest and the newest
+  # Python alone, 3.11 and 3.14, in the three parts, which hold every test but the nightly ones once; on 3.12 and 3.13
+  # it runs tests/unit alone, which needs no database, with no containers. The first part, or the unit part, lints too.
   test:
+    name: test (${{ matrix.python }}, ${{ matrix.part }})
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        python: ["3.11", "3.12", "3.13", "3.14"]
+        include: ${{ fromJSON((inputs.full || github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'full-ci')) && '[{"python":"3.11","part":"1/3"},{"python":"3.11","part":"2/3"},{"python":"3.11","part":"3/3"},{"python":"3.11","part":"nightly"},{"python":"3.12","part":"1/3"},{"python":"3.12","part":"2/3"},{"python":"3.12","part":"3/3"},{"python":"3.12","part":"nightly"},{"python":"3.13","part":"1/3"},{"python":"3.13","part":"2/3"},{"python":"3.13","part":"3/3"},{"python":"3.13","part":"nightly"},{"python":"3.14","part":"1/3"},{"python":"3.14","part":"2/3"},{"python":"3.14","part":"3/3"},{"python":"3.14","part":"nightly"}]' || '[{"python":"3.11","part":"1/3"},{"python":"3.11","part":"2/3"},{"python":"3.11","part":"3/3"},{"python":"3.14","part":"1/3"},{"python":"3.14","part":"2/3"},{"python":"3.14","part":"3/3"},{"python":"3.12","part":"unit"},{"python":"3.13","part":"unit"}]') }}
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v6
         with:
           python-version: ${{ matrix.python }}
       - run: uv sync
-      - run: uv run ruff check .
-      - run: uv run ruff format --check .
-      - run: uv run pyright
-      - run: uv run python scripts/unasync.py --check
-      - run: uv run pytest -q
+      - if: matrix.part == '1/3' || matrix.part == 'unit'
+        run: uv run ruff check .
+      - if: matrix.part == '1/3' || matrix.part == 'unit'
+        run: uv run ruff format --check .
+      - if: matrix.part == '1/3' || matrix.part == 'unit'
+        run: uv run pyright
+      - if: matrix.part == '1/3' || matrix.part == 'unit'
+        run: uv run python scripts/unasync.py --check
+      - name: Run the tests
+        env:
+          PART: ${{ matrix.part }}
+        run: |
+          case "$PART" in
+            nightly) picked=(-m nightly) ;;
+            unit) picked=(-o drivine_test_containers=false tests/unit) ;;
+            *) picked=(-m "not nightly" --shard "$PART") ;;
+          esac
+          uv run pytest -q --durations=15 --record-durations durations.json "${picked[@]}"
+      # How long each module took, to balance the parts by (tests/durations.json). From the slowest Python alone.
+      - if: always() && matrix.python == '3.11'
+        uses: actions/upload-artifact@v4
+        with:
+          name: durations-default-${{ strategy.job-index }}
+          path: durations.json
+          if-no-files-found: ignore
+          overwrite: true
 
-  # Oracle is left out of the job above: its tests run only with ORACLE_TESTS=true. This job runs what Oracle adds,
-  # on the newest Python alone: the tests whose name or parameters say oracle, which `-k oracle` picks.
-  test-oracle:
-    name: test (Oracle)
+  # What Oracle adds, on the newest Python alone, with ORACLE_TESTS=true: the tests whose name or parameters say oracle,
+  # which `-k oracle` picks. In four parts, and the nightly ones in a fifth on a full run.
+  oracle-part:
+    name: test (Oracle, ${{ matrix.part }})
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        part: ${{ fromJSON((inputs.full || github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'full-ci')) && '["1/4", "2/4", "3/4", "4/4", "nightly"]' || '["1/4", "2/4", "3/4", "4/4"]') }}
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v6
@@ -93,6 +140,29 @@ jobs:
         run: docker pull gvenzl/oracle-free:23-faststart
       - run: uv sync
       - name: Run the Oracle tests
-        run: uv run pytest -q -k oracle
         env:
           ORACLE_TESTS: "true"
+          PART: ${{ matrix.part }}
+        run: |
+          if [ "$PART" = nightly ]; then picked=(-m nightly); else picked=(-m "not nightly" --shard "$PART"); fi
+          uv run pytest -q -k oracle --durations=15 --record-durations durations.json "${picked[@]}"
+      - if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: durations-oracle-${{ strategy.job-index }}
+          path: durations.json
+          if-no-files-found: ignore
+          overwrite: true
+
+  # The check the single Oracle job gave before it was split: it passes when every part with Oracle passed. It runs
+  # whatever they concluded, as no job here may be skipped (see `tree`).
+  oracle:
+    name: test (Oracle)
+    needs: oracle-part
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Every part passed
+        run: |
+          echo "The parts with Oracle concluded ${{ needs.oracle-part.result }}."
+          [ "${{ needs.oracle-part.result }}" = success ]

.github/workflows/nightly.yml

@@ -0,0 +1,113 @@
+# Each night, the full CI on main: every test, the nightly ones a pull request leaves out among them (`-m nightly`), on
+# every store and every Python, and on the engines' latest images, without ci.yml's FalkorDB pin, so that an engine's
+# new release shows up here and not in a pull request's run. At 17:00 UTC, 03:00 in Brisbane.
+#
+# A failure opens an issue labelled `nightly`, or comments on the one open, naming the failed jobs, their failing
+# tests, and the commits since the last nightly that passed, among which is the one that broke it. A pass closes it.
+# A run on a branch other than main writes what it would say to its summary instead.
+name: Nightly
+
+on:
+  schedule:
+    - cron: "0 17 * * *"
+  workflow_dispatch:
+
+# Read only, so the tests run with a token that can't write to the repository; the report job asks for what it needs.
+permissions:
+  contents: read
+
+jobs:
+  ci:
+    uses: ./.github/workflows/ci.yml
+    with:
+      full: true
+
+  report:
+    needs: ci
+    if: always()
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      actions: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Open, update or close the nightly issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RESULT: ${{ needs.ci.result }}
+          EVENT: ${{ github.event_name }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          POST: ${{ github.ref == 'refs/heads/main' }}
+        run: |
+          repo="$GITHUB_REPOSITORY"
+          open=$(gh issue list --repo "$repo" --label nightly --state open --json number -q '.[0].number')
+          # Writes to the issue on main, and to the summary alone elsewhere. An issue's body or comment holds 65,536
+          # characters at most, and GitHub refuses a longer one, so the issue gets the first 60,000, and the summary
+          # the whole report.
+          post() {
+            echo "$1"
+            echo "$1" >> "$GITHUB_STEP_SUMMARY"
+            [ "$POST" = true ] || return 0
+            local text=$1
+            if [ "${#text}" -gt 60000 ]; then
+              text="${text:0:60000}"$'\n\n'"Cut short, as an issue holds 65,536 characters: the run's summary has the whole report, $RUN_URL"
+            fi
+            if [ "$2" = close ]; then
+              gh issue close "$open" --repo "$repo" --comment "$text"
+            elif [ -n "$open" ]; then
+              gh issue comment "$open" --repo "$repo" --body "$text"
+            else
+              gh label create nightly --repo "$repo" --color B60205 --description "The nightly CI failed" 2> /dev/null || true
+              gh issue create --repo "$repo" --label nightly --title "Nightly CI fails" --body "$text"
+            fi
+          }
+          # A run cancelled by hand says nothing of the tree. A scheduled run is never cancelled by hand, and a job past
+          # its timeout-minutes can conclude cancelled, so there a cancelled run or job is a hanging test: a failure.
+          [ "$RESULT" != cancelled ] || [ "$EVENT" = schedule ] || exit 0
+          picked='.conclusion == "failure" or .conclusion == "timed_out"'
+          [ "$EVENT" != schedule ] || picked="$picked or .conclusion == \"cancelled\""
+          if [ "$RESULT" = success ]; then
+            [ -z "$open" ] || post "The nightly passed again on $GITHUB_SHA: $RUN_URL" close
+            exit 0
+          fi
+          body=$(mktemp)
+          {
+            echo "The nightly on $GITHUB_SHA concluded $RESULT: $RUN_URL"
+            echo
+            gh api --paginate "repos/$repo/actions/runs/$GITHUB_RUN_ID/jobs?per_page=100" \
+              -q ".jobs[] | select($picked) | \"\\(.id)\\t\\(.name)\\t\\(.conclusion)\\t\\(.html_url)\"" |
+              while IFS=$'\t' read -r id name conclusion url; do
+                if [ "$conclusion" = cancelled ]; then
+                  echo "### [$name]($url), cancelled or timed out"
+                else
+                  echo "### [$name]($url)"
+                fi
+                echo
+                # pytest's short summary, a line for each failed test and each error, without the log's timestamps.
+                failing=$(gh api --allow-escape-sequences "repos/$repo/actions/jobs/$id/logs" |
+                  sed -E 's/^[^ ]+ //; s/\x1b\[[0-9;]*m//g' | grep -E '^(FAILED|ERROR) ' |
+                  cut -c1-300 | awk '!seen[$0]++' | head -40 || true)
+                if [ -n "$failing" ]; then
+                  printf '```\n%s\n```\n' "$failing"
+                elif [ "$conclusion" = cancelled ]; then
+                  echo "No failing test in its log: it was stopped before pytest's summary, at a test that hangs or past its time."
+                else
+                  echo "No failing test in its log: it failed in a step around the tests, or waits for jobs above."
+                fi
+                echo
+              done
+            green=$(gh api "repos/$repo/actions/workflows/nightly.yml/runs?branch=main&status=success&per_page=1" \
+              -q '.workflow_runs[0].head_sha // empty')
+            if [ -n "$green" ] && git cat-file -e "$green^{commit}" 2> /dev/null; then
+              echo "### Commits since the last nightly that passed, on $green"
+              echo
+              git log --format='- %h %s (%an)' "$green..$GITHUB_SHA"
+              [ -n "$(git log --format=%h "$green..$GITHUB_SHA")" ] || echo "None: the same commit passed before, so look at the images and the runners."
+            else
+              echo "No nightly has passed on main yet, so there are no commits to compare with."
+            fi
+          } > "$body"
+          post "$(cat "$body")"

.github/workflows/prepare-release.yml

@@ -181,7 +181,8 @@ jobs:
               echo "This PR was opened with GITHUB_TOKEN, so CI hasn't started on it. Push an empty commit to the branch, or close and reopen the PR, to start it."
             fi
           } > "$body"
-          url=$(gh pr create --repo "$REPO" --base "$DEFAULT_BRANCH" --head "$BRANCH" --title "Release $TAG" --body-file "$body")
+          # Labelled full-ci, so its CI runs the nightly tests too, and a release of its tree can trust that run.
+          url=$(gh pr create --repo "$REPO" --base "$DEFAULT_BRANCH" --head "$BRANCH" --title "Release $TAG" --body-file "$body" --label full-ci)
           echo "Opened $url" | tee -a "$GITHUB_STEP_SUMMARY"
           if [[ "$HAS_PR_TOKEN" != true ]]; then
             note="Opened with GITHUB_TOKEN, so CI doesn't run on it by itself. Push an empty commit to $BRANCH, or close and reopen the PR, to start it."

.github/workflows/release.yml

@@ -3,7 +3,7 @@
 # - A push to main that raises pyproject.toml's version (the merge of a "Release vX.Y.Z" PR) publishes it, and tags
 #   the commit vX.Y.Z. A push to main that changes pyproject.toml without raising the version publishes nothing.
 # - A push of a tag vX.Y.Z, by hand, publishes too: the fallback.
-# - Either way, the full CI runs, unless CI has already passed on the same tree; the distributions are built and
+# - Either way, the full CI runs, the nightly tests with it, unless a full CI run has already passed on the same tree; the distributions are built and
 #   checked, and those exact files are uploaded to PyPI with trusted publishing.
 # - workflow_dispatch, and a pull request that changes the release machinery, are dry runs: everything but the
 #   publish, with the built files left as the run's artifact to inspect.
@@ -95,9 +95,9 @@ jobs:
           echo "publish=$publish" >> "$GITHUB_OUTPUT"
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
 
-  # Whether ci.yml has already passed on the git tree this run builds, from a push or a pull request, so the gate
-  # needn't run it again: the release PR's run, when it was up to date with main and squash merged, or the push run on
-  # the merge commit, which starts beside this one and is waited for. A run on this tree that fails fails the release.
+  # Whether ci.yml has already passed on the git tree this run builds, from a push or a pull request, with the nightly
+  # tests, so the gate needn't run it again: the release PR's run (labelled full-ci), when it was up to date with main
+  # and squash merged, or the push run on the merge commit, which starts beside this one and is waited for. A run on this tree that fails fails the release.
   # scripts/ci_trust.py says how a run's tree is known, and why a pull request's run can be believed; the step summary
   # links the run trusted, or says why none was. Skipped, as the gate is, on a push that releases nothing.
   trust:
@@ -125,12 +125,15 @@ jobs:
           tree=$(git rev-parse 'HEAD^{tree}')
           python3 scripts/ci_trust.py "$GITHUB_REPOSITORY" "$tree" "$(git rev-parse HEAD)" "$PR_HEAD"
 
-  # The full CI, as ci.yml runs it on a push or a pull request, unless trust found it has already passed on this tree.
-  # A push that releases nothing stops at check: ci.yml runs on the same push to main by itself.
+  # The full CI, as ci.yml runs it on a push to main, with the nightly tests a pull request's run leaves out, unless
+  # trust found it has already passed on this tree. A push that releases nothing stops at check: ci.yml runs on the
+  # same push to main by itself.
   gate:
     needs: [check, trust]
     if: needs.trust.outputs.trusted != 'true'
     uses: ./.github/workflows/ci.yml
+    with:
+      full: true
 
   # Runs when the gate passed, or was skipped because trust found CI had passed on this tree. A failed or cancelled
   # check, trust or gate stops it, and so does a gate skipped for any other reason (a push that releases nothing).