Repository navigation
Release #150
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release | |
| on: | |
| schedule: | |
| # Monday-Wednesday at 2 AM UTC: alpha builds from dev | |
| - cron: "0 2 * * 1-3" | |
| # Thursday at 2 AM UTC: beta builds from dev | |
| - cron: "0 2 * * 4" | |
| workflow_dispatch: | |
| inputs: | |
| release_type: | |
| description: "Release type" | |
| required: true | |
| type: choice | |
| options: | |
| - alpha | |
| - beta | |
| - stable | |
| dry_run: | |
| description: "Dry run (skip PyPI publish)" | |
| required: false | |
| default: false | |
| type: boolean | |
| dry_run_ref: | |
| description: "Candidate ref to validate (allowed only with dry_run)" | |
| required: false | |
| default: "" | |
| type: string | |
| force_build: | |
| description: "Force build even if no changes" | |
| required: false | |
| default: false | |
| type: boolean | |
| version_override: | |
| description: "Override stable version or prerelease base (e.g. 4.4.0)" | |
| required: false | |
| type: string | |
| concurrency: | |
| group: release-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| # ── 1. Determine release type and check for changes ────────────────── | |
| determine-release: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| release_type: ${{ steps.resolve.outputs.release_type }} | |
| has_changes: ${{ steps.changes.outputs.has_changes }} | |
| target_branch: ${{ steps.resolve.outputs.target_branch }} | |
| source_sha: ${{ steps.source.outputs.sha }} | |
| steps: | |
| - name: Resolve release type | |
| id: resolve | |
| env: | |
| DRY_RUN: ${{ inputs.dry_run }} | |
| DRY_RUN_REF: ${{ inputs.dry_run_ref }} | |
| run: | | |
| if [ -n "$DRY_RUN_REF" ] && [ "$DRY_RUN" != "true" ]; then | |
| echo "dry_run_ref is allowed only with dry_run=true" | |
| exit 1 | |
| fi | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| TYPE="${{ inputs.release_type }}" | |
| elif [ "${{ github.event.schedule }}" = "0 2 * * 4" ]; then | |
| TYPE="beta" | |
| else | |
| TYPE="alpha" | |
| fi | |
| if [ "$TYPE" = "stable" ]; then | |
| BRANCH="main" | |
| else | |
| BRANCH="dev" | |
| fi | |
| { | |
| echo "release_type=$TYPE" | |
| echo "target_branch=$BRANCH" | |
| echo "checkout_ref=${DRY_RUN_REF:-$BRANCH}" | |
| } >> "$GITHUB_OUTPUT" | |
| echo "Release type: $TYPE from $BRANCH" | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ steps.resolve.outputs.checkout_ref }} | |
| - name: Freeze source commit | |
| id: source | |
| run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" | |
| - name: Check for changes | |
| id: changes | |
| run: | | |
| TYPE="${{ steps.resolve.outputs.release_type }}" | |
| if [ "$TYPE" = "alpha" ]; then | |
| LAST_TAG=$(git tag -l "*a*" --sort=-version:refname | head -n1) | |
| elif [ "$TYPE" = "beta" ]; then | |
| LAST_TAG=$(git tag -l "*b*" --sort=-version:refname | head -n1) | |
| else | |
| LAST_TAG=$(git describe --tags --abbrev=0 --match "v[0-9]*.[0-9]*.[0-9]*" 2>/dev/null || echo "") | |
| fi | |
| if [ -z "$LAST_TAG" ]; then | |
| COMMIT_COUNT=$(git rev-list --count --since="7 days ago" HEAD) | |
| else | |
| COMMIT_COUNT=$(git rev-list --count "${LAST_TAG}..HEAD") | |
| fi | |
| echo "Commits since ${LAST_TAG:-'(none)'}: $COMMIT_COUNT" | |
| if [ "$COMMIT_COUNT" -gt 0 ] || [ "${{ inputs.force_build }}" = "true" ]; then | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_changes=false" >> "$GITHUB_OUTPUT" | |
| echo "No changes detected, skipping release" | |
| fi | |
| # ── 2. Test gate ────────────────────────────────────────────────────── | |
| test: | |
| needs: determine-release | |
| if: needs.determine-release.outputs.has_changes == 'true' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ needs.determine-release.outputs.source_sha }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "pip" | |
| - name: Install Tesseract OCR | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y tesseract-ocr libtesseract-dev | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[all,test]" | |
| pip install -r requirements-test.txt | |
| python -m spacy download en_core_web_lg | |
| datafog download-model urchade/gliner_multi_pii-v1 --engine gliner | |
| - name: Run release tests | |
| env: | |
| OMP_NUM_THREADS: "1" | |
| MKL_NUM_THREADS: "1" | |
| OPENBLAS_NUM_THREADS: "1" | |
| run: | | |
| python -m pytest tests/ -m "not slow" \ | |
| --ignore=tests/test_detection_accuracy.py \ | |
| --ignore=tests/test_image_service.py \ | |
| --ignore=tests/test_ocr_integration.py \ | |
| --ignore=tests/test_spark_integration.py \ | |
| --cov=datafog --cov-report=xml --cov-config=.coveragerc | |
| - name: Run detection accuracy corpus | |
| run: python -m pytest tests/test_detection_accuracy.py -v --tb=short | |
| - name: Run performance validation | |
| run: | | |
| OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 python tests/simple_performance_test.py | |
| # ── 3. Build & Publish ──────────────────────────────────────────────── | |
| python314-core: | |
| needs: determine-release | |
| if: needs.determine-release.outputs.has_changes == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ needs.determine-release.outputs.source_sha }} | |
| - name: Set up Python 3.14 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.14" | |
| cache: "pip" | |
| - name: Install core + CLI dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[test,cli]" -r requirements-test.txt | |
| - name: Run Python 3.14 core + CLI tests | |
| run: | | |
| pytest tests/ \ | |
| -m "not slow" \ | |
| --ignore=tests/test_gliner_annotator.py \ | |
| --ignore=tests/test_image_service.py \ | |
| --ignore=tests/test_ocr_integration.py \ | |
| --ignore=tests/test_spark_integration.py \ | |
| --ignore=tests/test_text_service_integration.py | |
| rust-bridge: | |
| needs: determine-release | |
| if: needs.determine-release.outputs.has_changes == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: ubuntu-latest | |
| python-version: "3.10" | |
| - os: ubuntu-latest | |
| python-version: "3.14" | |
| - os: macos-latest | |
| python-version: "3.12" | |
| - os: windows-latest | |
| python-version: "3.12" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ needs.determine-release.outputs.source_sha }} | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Build and install wheel with published Core | |
| shell: bash | |
| run: | | |
| python -m pip install build | |
| python -m build --wheel | |
| python -c "import glob, subprocess, sys; wheel = glob.glob('dist/*.whl')[0]; subprocess.check_call([sys.executable, '-m', 'pip', 'install', wheel + '[test,cli,rust]'])" | |
| python -m pip check | |
| - name: Verify installed wheel | |
| run: python -I scripts/check_rust_install.py | |
| - name: Test binding integration and compatibility | |
| run: python -m pytest tests/test_contract_481.py tests/test_rust_backend.py tests/test_api_bridge_49.py tests/test_rust_contract.py tests/test_core_capability_adapter.py tests/test_core04_integration.py -q | |
| - name: Record parity | |
| run: python -m tests.rust_contract --output rust-parity.json | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: release-rust-parity-${{ matrix.os }}-${{ matrix.python-version }} | |
| path: rust-parity.json | |
| publish: | |
| needs: [determine-release, test, python314-core, rust-bridge] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| version: ${{ steps.version.outputs.version }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ needs.determine-release.outputs.source_sha }} | |
| token: ${{ secrets.GH_PAT }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install build twine bump2version | |
| - name: Configure git | |
| if: inputs.dry_run != true | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| - name: Generate version | |
| id: version | |
| env: | |
| VERSION_OVERRIDE: ${{ inputs.version_override }} | |
| run: | | |
| set -e | |
| git fetch --tags | |
| TYPE="${{ needs.determine-release.outputs.release_type }}" | |
| CURRENT=$(sed -n 's/^__version__ = "\([^"]*\)"/\1/p' datafog/__about__.py) | |
| if [ -z "$CURRENT" ]; then | |
| echo "Failed to parse current version from datafog/__about__.py" | |
| exit 1 | |
| fi | |
| echo "Current version: $CURRENT" | |
| # Strip any pre-release suffix to get base version | |
| BASE=$(echo "$CURRENT" | sed -E 's/(a|b)[0-9]+([.][0-9A-Za-z]+)?$//') | |
| if [ -n "$VERSION_OVERRIDE" ]; then | |
| BASE="$VERSION_OVERRIDE" | |
| if echo "$BASE" | grep -Eq '(a|b)[0-9]+([.][0-9A-Za-z]+)?$'; then | |
| echo "version_override must be a stable base version like 4.4.0, not a prerelease" | |
| exit 1 | |
| fi | |
| fi | |
| if ! echo "$BASE" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then | |
| echo "Release base must have numeric major.minor.patch format" | |
| exit 1 | |
| fi | |
| echo "Base version: $BASE" | |
| if [ "$TYPE" = "alpha" ]; then | |
| ALPHA_NUM=1 | |
| while git tag -l "v${BASE}a${ALPHA_NUM}" | grep -q .; do | |
| ALPHA_NUM=$((ALPHA_NUM + 1)) | |
| done | |
| VERSION="${BASE}a${ALPHA_NUM}" | |
| elif [ "$TYPE" = "beta" ]; then | |
| BETA_NUM=1 | |
| while git tag -l "v${BASE}b${BETA_NUM}" | grep -q .; do | |
| BETA_NUM=$((BETA_NUM + 1)) | |
| done | |
| VERSION="${BASE}b${BETA_NUM}" | |
| else | |
| VERSION="$BASE" | |
| fi | |
| echo "version=$VERSION" >> "$GITHUB_OUTPUT" | |
| echo "Publishing version: $VERSION" | |
| sed -i "s/__version__ = \".*\"/__version__ = \"$VERSION\"/" datafog/__about__.py | |
| if grep -q 'version="' setup.py 2>/dev/null; then | |
| sed -i "s/version=\".*\"/version=\"$VERSION\"/" setup.py | |
| fi | |
| - name: Generate changelog | |
| env: | |
| VERSION: ${{ steps.version.outputs.version }} | |
| run: | | |
| TYPE="${{ needs.determine-release.outputs.release_type }}" | |
| if [ "$TYPE" = "alpha" ]; then | |
| python scripts/generate_changelog.py --alpha --output RELEASE_CHANGELOG.md | |
| elif [ "$TYPE" = "beta" ]; then | |
| python scripts/generate_changelog.py --beta --output RELEASE_CHANGELOG.md | |
| elif [ -f "RELEASE_NOTES_${VERSION}.md" ]; then | |
| cp "RELEASE_NOTES_${VERSION}.md" RELEASE_CHANGELOG.md | |
| else | |
| python scripts/generate_changelog.py --output RELEASE_CHANGELOG.md | |
| fi | |
| - name: Build package | |
| run: | | |
| python -m build | |
| python scripts/check_wheel_size.py | |
| python -m twine check dist/* | |
| - name: Verify final versioned wheel with published Core | |
| run: | | |
| python -c "import glob, subprocess, sys; wheel = glob.glob('dist/*.whl')[0]; subprocess.check_call([sys.executable, '-m', 'pip', 'install', wheel + '[rust]'])" | |
| python -m pip check | |
| python -I scripts/check_rust_install.py | |
| - name: Save release artifacts for review | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: release-${{ steps.version.outputs.version }}-${{ needs.determine-release.outputs.source_sha }} | |
| path: | | |
| dist/* | |
| RELEASE_CHANGELOG.md | |
| if-no-files-found: error | |
| - name: Verify release branch still matches tested source | |
| if: inputs.dry_run != true | |
| env: | |
| RELEASE_BRANCH: ${{ needs.determine-release.outputs.target_branch }} | |
| TESTED_SHA: ${{ needs.determine-release.outputs.source_sha }} | |
| run: | | |
| git fetch origin "$RELEASE_BRANCH" | |
| if [ "$(git rev-parse FETCH_HEAD)" != "$TESTED_SHA" ]; then | |
| echo "Release branch changed during validation; rerun against its new head" | |
| exit 1 | |
| fi | |
| - name: Publish to PyPI | |
| if: inputs.dry_run != true | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} | |
| run: | | |
| python -m twine upload dist/* --verbose | |
| - name: Commit version bump & create release | |
| if: inputs.dry_run != true | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_PAT }} | |
| run: | | |
| VERSION="${{ steps.version.outputs.version }}" | |
| TYPE="${{ needs.determine-release.outputs.release_type }}" | |
| BRANCH="${{ needs.determine-release.outputs.target_branch }}" | |
| git add datafog/__about__.py setup.py | |
| git commit -m "chore: bump version to $VERSION [skip ci]" || echo "No version changes to commit" | |
| git push origin "HEAD:$BRANCH" | |
| git tag -a "v$VERSION" -m "Release $VERSION" | |
| git push origin "v$VERSION" | |
| PRERELEASE_FLAG="" | |
| TITLE="" | |
| if [ "$TYPE" = "alpha" ]; then | |
| PRERELEASE_FLAG="--prerelease" | |
| TITLE="Nightly Alpha $VERSION" | |
| elif [ "$TYPE" = "beta" ]; then | |
| PRERELEASE_FLAG="--prerelease" | |
| TITLE="Beta Release $VERSION" | |
| else | |
| TITLE="DataFog v$VERSION" | |
| fi | |
| gh release create "v$VERSION" \ | |
| --title "$TITLE" \ | |
| --notes-file RELEASE_CHANGELOG.md \ | |
| $PRERELEASE_FLAG \ | |
| --target "$BRANCH" \ | |
| dist/* | |
| - name: Dry run summary | |
| if: inputs.dry_run == true | |
| run: | | |
| echo "DRY RUN COMPLETE" | |
| echo "Would have published: ${{ steps.version.outputs.version }}" | |
| echo "Package contents:" | |
| ls -la dist/ | |
| # ── 4. Cleanup old pre-releases ─────────────────────────────────────── | |
| cleanup: | |
| needs: [determine-release, publish] | |
| if: needs.determine-release.outputs.release_type != 'stable' && inputs.dry_run != true | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Prune old alpha releases (keep 7) | |
| if: needs.determine-release.outputs.release_type == 'alpha' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_PAT }} | |
| run: | | |
| echo "Cleaning up old alpha releases (keep last 7)..." | |
| ALPHA_RELEASES=$(gh release list --limit 50 | grep -i alpha | tail -n +8 | cut -f3) | |
| for release in $ALPHA_RELEASES; do | |
| echo "Deleting old alpha release: $release" | |
| gh release delete "$release" --yes || true | |
| git push --delete origin "$release" 2>/dev/null || true | |
| done | |
| - name: Prune old beta releases (keep 5) | |
| if: needs.determine-release.outputs.release_type == 'beta' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_PAT }} | |
| run: | | |
| echo "Cleaning up old beta releases (keep last 5)..." | |
| BETA_RELEASES=$(gh release list --limit 30 | grep -i beta | tail -n +6 | cut -f3) | |
| for release in $BETA_RELEASES; do | |
| echo "Deleting old beta release: $release" | |
| gh release delete "$release" --yes || true | |
| git push --delete origin "$release" 2>/dev/null || true | |
| done |