release.yml

name: Release

on:
  schedule:
    # Monday-Wednesday at 2 AM UTC: alpha builds from dev
    - cron: "0 2 * * 1-3"
    # Thursday at 2 AM UTC: beta builds from dev
    - cron: "0 2 * * 4"
  workflow_dispatch:
    inputs:
      release_type:
        description: "Release type"
        required: true
        type: choice
        options:
          - alpha
          - beta
          - stable
      dry_run:
        description: "Dry run (skip PyPI publish)"
        required: false
        default: false
        type: boolean
      dry_run_ref:
        description: "Candidate ref to validate (allowed only with dry_run)"
        required: false
        default: ""
        type: string
      force_build:
        description: "Force build even if no changes"
        required: false
        default: false
        type: boolean
      version_override:
        description: "Override stable version or prerelease base (e.g. 4.4.0)"
        required: false
        type: string

concurrency:
  group: release-${{ github.ref }}
  cancel-in-progress: false

jobs:
  # ── 1. Determine release type and check for changes ──────────────────
  determine-release:
    runs-on: ubuntu-latest
    outputs:
      release_type: ${{ steps.resolve.outputs.release_type }}
      has_changes: ${{ steps.changes.outputs.has_changes }}
      target_branch: ${{ steps.resolve.outputs.target_branch }}
      source_sha: ${{ steps.source.outputs.sha }}
    steps:
      - name: Resolve release type
        id: resolve
        env:
          DRY_RUN: ${{ inputs.dry_run }}
          DRY_RUN_REF: ${{ inputs.dry_run_ref }}
        run: |
          if [ -n "$DRY_RUN_REF" ] && [ "$DRY_RUN" != "true" ]; then
            echo "dry_run_ref is allowed only with dry_run=true"
            exit 1
          fi
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            TYPE="${{ inputs.release_type }}"
          elif [ "${{ github.event.schedule }}" = "0 2 * * 4" ]; then
            TYPE="beta"
          else
            TYPE="alpha"
          fi

          if [ "$TYPE" = "stable" ]; then
            BRANCH="main"
          else
            BRANCH="dev"
          fi

          {
            echo "release_type=$TYPE"
            echo "target_branch=$BRANCH"
            echo "checkout_ref=${DRY_RUN_REF:-$BRANCH}"
          } >> "$GITHUB_OUTPUT"
          echo "Release type: $TYPE from $BRANCH"

      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          ref: ${{ steps.resolve.outputs.checkout_ref }}

      - name: Freeze source commit
        id: source
        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"

      - name: Check for changes
        id: changes
        run: |
          TYPE="${{ steps.resolve.outputs.release_type }}"

          if [ "$TYPE" = "alpha" ]; then
            LAST_TAG=$(git tag -l "*a*" --sort=-version:refname | head -n1)
          elif [ "$TYPE" = "beta" ]; then
            LAST_TAG=$(git tag -l "*b*" --sort=-version:refname | head -n1)
          else
            LAST_TAG=$(git describe --tags --abbrev=0 --match "v[0-9]*.[0-9]*.[0-9]*" 2>/dev/null || echo "")
          fi

          if [ -z "$LAST_TAG" ]; then
            COMMIT_COUNT=$(git rev-list --count --since="7 days ago" HEAD)
          else
            COMMIT_COUNT=$(git rev-list --count "${LAST_TAG}..HEAD")
          fi

          echo "Commits since ${LAST_TAG:-'(none)'}: $COMMIT_COUNT"

          if [ "$COMMIT_COUNT" -gt 0 ] || [ "${{ inputs.force_build }}" = "true" ]; then
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
          else
            echo "has_changes=false" >> "$GITHUB_OUTPUT"
            echo "No changes detected, skipping release"
          fi

  # ── 2. Test gate ──────────────────────────────────────────────────────
  test:
    needs: determine-release
    if: needs.determine-release.outputs.has_changes == 'true'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12"]
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          ref: ${{ needs.determine-release.outputs.source_sha }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"

      - name: Install Tesseract OCR
        run: |
          sudo apt-get update
          sudo apt-get install -y tesseract-ocr libtesseract-dev

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[all,test]"
          pip install -r requirements-test.txt
          python -m spacy download en_core_web_lg
          datafog download-model urchade/gliner_multi_pii-v1 --engine gliner

      - name: Run release tests
        env:
          OMP_NUM_THREADS: "1"
          MKL_NUM_THREADS: "1"
          OPENBLAS_NUM_THREADS: "1"
        run: |
          python -m pytest tests/ -m "not slow" \
            --ignore=tests/test_detection_accuracy.py \
            --ignore=tests/test_image_service.py \
            --ignore=tests/test_ocr_integration.py \
            --ignore=tests/test_spark_integration.py \
            --cov=datafog --cov-report=xml --cov-config=.coveragerc

      - name: Run detection accuracy corpus
        run: python -m pytest tests/test_detection_accuracy.py -v --tb=short

      - name: Run performance validation
        run: |
          OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 python tests/simple_performance_test.py

  # ── 3. Build & Publish ────────────────────────────────────────────────
  python314-core:
    needs: determine-release
    if: needs.determine-release.outputs.has_changes == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          ref: ${{ needs.determine-release.outputs.source_sha }}

      - name: Set up Python 3.14
        uses: actions/setup-python@v6
        with:
          python-version: "3.14"
          cache: "pip"

      - name: Install core + CLI dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[test,cli]" -r requirements-test.txt

      - name: Run Python 3.14 core + CLI tests
        run: |
          pytest tests/ \
            -m "not slow" \
            --ignore=tests/test_gliner_annotator.py \
            --ignore=tests/test_image_service.py \
            --ignore=tests/test_ocr_integration.py \
            --ignore=tests/test_spark_integration.py \
            --ignore=tests/test_text_service_integration.py

  rust-bridge:
    needs: determine-release
    if: needs.determine-release.outputs.has_changes == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            python-version: "3.10"
          - os: ubuntu-latest
            python-version: "3.14"
          - os: macos-latest
            python-version: "3.12"
          - os: windows-latest
            python-version: "3.12"
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ needs.determine-release.outputs.source_sha }}
      - uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
      - name: Build and install wheel with published Core
        shell: bash
        run: |
          python -m pip install build
          python -m build --wheel
          python -c "import glob, subprocess, sys; wheel = glob.glob('dist/*.whl')[0]; subprocess.check_call([sys.executable, '-m', 'pip', 'install', wheel + '[test,cli,rust]'])"
          python -m pip check
      - name: Verify installed wheel
        run: python -I scripts/check_rust_install.py
      - name: Test binding integration and compatibility
        run: python -m pytest tests/test_contract_481.py tests/test_rust_backend.py tests/test_api_bridge_49.py tests/test_rust_contract.py tests/test_core_capability_adapter.py tests/test_core04_integration.py -q
      - name: Record parity
        run: python -m tests.rust_contract --output rust-parity.json
      - uses: actions/upload-artifact@v4
        with:
          name: release-rust-parity-${{ matrix.os }}-${{ matrix.python-version }}
          path: rust-parity.json

  publish:
    needs: [determine-release, test, python314-core, rust-bridge]
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.version.outputs.version }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          ref: ${{ needs.determine-release.outputs.source_sha }}
          token: ${{ secrets.GH_PAT }}

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install build twine bump2version

      - name: Configure git
        if: inputs.dry_run != true
        run: |
          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"

      - name: Generate version
        id: version
        env:
          VERSION_OVERRIDE: ${{ inputs.version_override }}
        run: |
          set -e
          git fetch --tags

          TYPE="${{ needs.determine-release.outputs.release_type }}"
          CURRENT=$(sed -n 's/^__version__ = "\([^"]*\)"/\1/p' datafog/__about__.py)
          if [ -z "$CURRENT" ]; then
            echo "Failed to parse current version from datafog/__about__.py"
            exit 1
          fi
          echo "Current version: $CURRENT"

          # Strip any pre-release suffix to get base version
          BASE=$(echo "$CURRENT" | sed -E 's/(a|b)[0-9]+([.][0-9A-Za-z]+)?$//')
          if [ -n "$VERSION_OVERRIDE" ]; then
            BASE="$VERSION_OVERRIDE"
            if echo "$BASE" | grep -Eq '(a|b)[0-9]+([.][0-9A-Za-z]+)?$'; then
              echo "version_override must be a stable base version like 4.4.0, not a prerelease"
              exit 1
            fi
          fi
          if ! echo "$BASE" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then
            echo "Release base must have numeric major.minor.patch format"
            exit 1
          fi
          echo "Base version: $BASE"

          if [ "$TYPE" = "alpha" ]; then
            ALPHA_NUM=1
            while git tag -l "v${BASE}a${ALPHA_NUM}" | grep -q .; do
              ALPHA_NUM=$((ALPHA_NUM + 1))
            done
            VERSION="${BASE}a${ALPHA_NUM}"

          elif [ "$TYPE" = "beta" ]; then
            BETA_NUM=1
            while git tag -l "v${BASE}b${BETA_NUM}" | grep -q .; do
              BETA_NUM=$((BETA_NUM + 1))
            done
            VERSION="${BASE}b${BETA_NUM}"

          else
            VERSION="$BASE"
          fi

          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
          echo "Publishing version: $VERSION"

          sed -i "s/__version__ = \".*\"/__version__ = \"$VERSION\"/" datafog/__about__.py
          if grep -q 'version="' setup.py 2>/dev/null; then
            sed -i "s/version=\".*\"/version=\"$VERSION\"/" setup.py
          fi

      - name: Generate changelog
        env:
          VERSION: ${{ steps.version.outputs.version }}
        run: |
          TYPE="${{ needs.determine-release.outputs.release_type }}"
          if [ "$TYPE" = "alpha" ]; then
            python scripts/generate_changelog.py --alpha --output RELEASE_CHANGELOG.md
          elif [ "$TYPE" = "beta" ]; then
            python scripts/generate_changelog.py --beta --output RELEASE_CHANGELOG.md
          elif [ -f "RELEASE_NOTES_${VERSION}.md" ]; then
            cp "RELEASE_NOTES_${VERSION}.md" RELEASE_CHANGELOG.md
          else
            python scripts/generate_changelog.py --output RELEASE_CHANGELOG.md
          fi

      - name: Build package
        run: |
          python -m build
          python scripts/check_wheel_size.py
          python -m twine check dist/*

      - name: Verify final versioned wheel with published Core
        run: |
          python -c "import glob, subprocess, sys; wheel = glob.glob('dist/*.whl')[0]; subprocess.check_call([sys.executable, '-m', 'pip', 'install', wheel + '[rust]'])"
          python -m pip check
          python -I scripts/check_rust_install.py

      - name: Save release artifacts for review
        uses: actions/upload-artifact@v4
        with:
          name: release-${{ steps.version.outputs.version }}-${{ needs.determine-release.outputs.source_sha }}
          path: |
            dist/*
            RELEASE_CHANGELOG.md
          if-no-files-found: error

      - name: Verify release branch still matches tested source
        if: inputs.dry_run != true
        env:
          RELEASE_BRANCH: ${{ needs.determine-release.outputs.target_branch }}
          TESTED_SHA: ${{ needs.determine-release.outputs.source_sha }}
        run: |
          git fetch origin "$RELEASE_BRANCH"
          if [ "$(git rev-parse FETCH_HEAD)" != "$TESTED_SHA" ]; then
            echo "Release branch changed during validation; rerun against its new head"
            exit 1
          fi

      - name: Publish to PyPI
        if: inputs.dry_run != true
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
        run: |
          python -m twine upload dist/* --verbose

      - name: Commit version bump & create release
        if: inputs.dry_run != true
        env:
          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
        run: |
          VERSION="${{ steps.version.outputs.version }}"
          TYPE="${{ needs.determine-release.outputs.release_type }}"
          BRANCH="${{ needs.determine-release.outputs.target_branch }}"

          git add datafog/__about__.py setup.py
          git commit -m "chore: bump version to $VERSION [skip ci]" || echo "No version changes to commit"
          git push origin "HEAD:$BRANCH"

          git tag -a "v$VERSION" -m "Release $VERSION"
          git push origin "v$VERSION"

          PRERELEASE_FLAG=""
          TITLE=""
          if [ "$TYPE" = "alpha" ]; then
            PRERELEASE_FLAG="--prerelease"
            TITLE="Nightly Alpha $VERSION"
          elif [ "$TYPE" = "beta" ]; then
            PRERELEASE_FLAG="--prerelease"
            TITLE="Beta Release $VERSION"
          else
            TITLE="DataFog v$VERSION"
          fi

          gh release create "v$VERSION" \
            --title "$TITLE" \
            --notes-file RELEASE_CHANGELOG.md \
            $PRERELEASE_FLAG \
            --target "$BRANCH" \
            dist/*

      - name: Dry run summary
        if: inputs.dry_run == true
        run: |
          echo "DRY RUN COMPLETE"
          echo "Would have published: ${{ steps.version.outputs.version }}"
          echo "Package contents:"
          ls -la dist/

  # ── 4. Cleanup old pre-releases ───────────────────────────────────────
  cleanup:
    needs: [determine-release, publish]
    if: needs.determine-release.outputs.release_type != 'stable' && inputs.dry_run != true
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Prune old alpha releases (keep 7)
        if: needs.determine-release.outputs.release_type == 'alpha'
        env:
          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
        run: |
          echo "Cleaning up old alpha releases (keep last 7)..."
          ALPHA_RELEASES=$(gh release list --limit 50 | grep -i alpha | tail -n +8 | cut -f3)

          for release in $ALPHA_RELEASES; do
            echo "Deleting old alpha release: $release"
            gh release delete "$release" --yes || true
            git push --delete origin "$release" 2>/dev/null || true
          done

      - name: Prune old beta releases (keep 5)
        if: needs.determine-release.outputs.release_type == 'beta'
        env:
          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
        run: |
          echo "Cleaning up old beta releases (keep last 5)..."
          BETA_RELEASES=$(gh release list --limit 30 | grep -i beta | tail -n +6 | cut -f3)

          for release in $BETA_RELEASES; do
            echo "Deleting old beta release: $release"
            gh release delete "$release" --yes || true
            git push --delete origin "$release" 2>/dev/null || true
          done