diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c39cd405..e72c9e2a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.7.0 +current_version = 4.9.0 commit = True tag = True tag_name = v{new_version} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 30bd23b4..77e7aedd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,6 +21,11 @@ on: required: false default: false type: boolean + dry_run_ref: + description: "Candidate ref to validate (allowed only with dry_run)" + required: false + default: "" + type: string force_build: description: "Force build even if no changes" required: false @@ -43,10 +48,18 @@ jobs: release_type: ${{ steps.resolve.outputs.release_type }} has_changes: ${{ steps.changes.outputs.has_changes }} target_branch: ${{ steps.resolve.outputs.target_branch }} + source_sha: ${{ steps.source.outputs.sha }} steps: - name: Resolve release type id: resolve + env: + DRY_RUN: ${{ inputs.dry_run }} + DRY_RUN_REF: ${{ inputs.dry_run_ref }} run: | + if [ -n "$DRY_RUN_REF" ] && [ "$DRY_RUN" != "true" ]; then + echo "dry_run_ref is allowed only with dry_run=true" + exit 1 + fi if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then TYPE="${{ inputs.release_type }}" elif [ "${{ github.event.schedule }}" = "0 2 * * 4" ]; then @@ -61,14 +74,21 @@ jobs: BRANCH="dev" fi - echo "release_type=$TYPE" >> "$GITHUB_OUTPUT" - echo "target_branch=$BRANCH" >> "$GITHUB_OUTPUT" + { + echo "release_type=$TYPE" + echo "target_branch=$BRANCH" + echo "checkout_ref=${DRY_RUN_REF:-$BRANCH}" + } >> "$GITHUB_OUTPUT" echo "Release type: $TYPE from $BRANCH" - uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ steps.resolve.outputs.target_branch }} + ref: ${{ steps.resolve.outputs.checkout_ref }} + + - name: Freeze source commit + id: source + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - name: Check for changes id: changes @@ -110,7 +130,7 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ needs.determine-release.outputs.target_branch }} + ref: ${{ needs.determine-release.outputs.source_sha }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 @@ -131,9 +151,21 @@ jobs: python -m spacy download en_core_web_lg datafog download-model urchade/gliner_multi_pii-v1 --engine gliner - - name: Run tests with segfault protection + - name: Run release tests + env: + OMP_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" run: | - python run_tests.py tests/ --ignore=tests/test_gliner_annotator.py --cov-report=xml --cov-config=.coveragerc + python -m pytest tests/ -m "not slow" \ + --ignore=tests/test_detection_accuracy.py \ + --ignore=tests/test_image_service.py \ + --ignore=tests/test_ocr_integration.py \ + --ignore=tests/test_spark_integration.py \ + --cov=datafog --cov-report=xml --cov-config=.coveragerc + + - name: Run detection accuracy corpus + run: python -m pytest tests/test_detection_accuracy.py -v --tb=short - name: Run performance validation run: | @@ -148,7 +180,7 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ needs.determine-release.outputs.target_branch }} + ref: ${{ needs.determine-release.outputs.source_sha }} - name: Set up Python 3.14 uses: actions/setup-python@v6 @@ -171,8 +203,49 @@ jobs: --ignore=tests/test_spark_integration.py \ --ignore=tests/test_text_service_integration.py + rust-bridge: + needs: determine-release + if: needs.determine-release.outputs.has_changes == 'true' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python-version: "3.10" + - os: ubuntu-latest + python-version: "3.14" + - os: macos-latest + python-version: "3.12" + - os: windows-latest + python-version: "3.12" + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.determine-release.outputs.source_sha }} + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Build and install wheel with published Core + shell: bash + run: | + python -m pip install build + python -m build --wheel + python -c "import glob, subprocess, sys; wheel = glob.glob('dist/*.whl')[0]; subprocess.check_call([sys.executable, '-m', 'pip', 'install', wheel + '[test,cli,rust]'])" + python -m pip check + - name: Verify installed wheel + run: python -I scripts/check_rust_install.py + - name: Test binding integration and compatibility + run: python -m pytest tests/test_contract_481.py tests/test_rust_backend.py tests/test_api_bridge_49.py tests/test_rust_contract.py tests/test_core_capability_adapter.py tests/test_core04_integration.py -q + - name: Record parity + run: python -m tests.rust_contract --output rust-parity.json + - uses: actions/upload-artifact@v4 + with: + name: release-rust-parity-${{ matrix.os }}-${{ matrix.python-version }} + path: rust-parity.json + publish: - needs: [determine-release, test, python314-core] + needs: [determine-release, test, python314-core, rust-bridge] runs-on: ubuntu-latest outputs: version: ${{ steps.version.outputs.version }} @@ -180,7 +253,7 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ needs.determine-release.outputs.target_branch }} + ref: ${{ needs.determine-release.outputs.source_sha }} token: ${{ secrets.GH_PAT }} - name: Set up Python @@ -194,12 +267,15 @@ jobs: pip install build twine bump2version - name: Configure git + if: inputs.dry_run != true run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" - name: Generate version id: version + env: + VERSION_OVERRIDE: ${{ inputs.version_override }} run: | set -e git fetch --tags @@ -214,13 +290,17 @@ jobs: # Strip any pre-release suffix to get base version BASE=$(echo "$CURRENT" | sed -E 's/(a|b)[0-9]+([.][0-9A-Za-z]+)?$//') - if [ -n "${{ inputs.version_override }}" ]; then - BASE="${{ inputs.version_override }}" + if [ -n "$VERSION_OVERRIDE" ]; then + BASE="$VERSION_OVERRIDE" if echo "$BASE" | grep -Eq '(a|b)[0-9]+([.][0-9A-Za-z]+)?$'; then echo "version_override must be a stable base version like 4.4.0, not a prerelease" exit 1 fi fi + if ! echo "$BASE" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "Release base must have numeric major.minor.patch format" + exit 1 + fi echo "Base version: $BASE" if [ "$TYPE" = "alpha" ]; then @@ -250,12 +330,16 @@ jobs: fi - name: Generate changelog + env: + VERSION: ${{ steps.version.outputs.version }} run: | TYPE="${{ needs.determine-release.outputs.release_type }}" if [ "$TYPE" = "alpha" ]; then python scripts/generate_changelog.py --alpha --output RELEASE_CHANGELOG.md elif [ "$TYPE" = "beta" ]; then python scripts/generate_changelog.py --beta --output RELEASE_CHANGELOG.md + elif [ -f "RELEASE_NOTES_${VERSION}.md" ]; then + cp "RELEASE_NOTES_${VERSION}.md" RELEASE_CHANGELOG.md else python scripts/generate_changelog.py --output RELEASE_CHANGELOG.md fi @@ -264,6 +348,34 @@ jobs: run: | python -m build python scripts/check_wheel_size.py + python -m twine check dist/* + + - name: Verify final versioned wheel with published Core + run: | + python -c "import glob, subprocess, sys; wheel = glob.glob('dist/*.whl')[0]; subprocess.check_call([sys.executable, '-m', 'pip', 'install', wheel + '[rust]'])" + python -m pip check + python -I scripts/check_rust_install.py + + - name: Save release artifacts for review + uses: actions/upload-artifact@v4 + with: + name: release-${{ steps.version.outputs.version }}-${{ needs.determine-release.outputs.source_sha }} + path: | + dist/* + RELEASE_CHANGELOG.md + if-no-files-found: error + + - name: Verify release branch still matches tested source + if: inputs.dry_run != true + env: + RELEASE_BRANCH: ${{ needs.determine-release.outputs.target_branch }} + TESTED_SHA: ${{ needs.determine-release.outputs.source_sha }} + run: | + git fetch origin "$RELEASE_BRANCH" + if [ "$(git rev-parse FETCH_HEAD)" != "$TESTED_SHA" ]; then + echo "Release branch changed during validation; rerun against its new head" + exit 1 + fi - name: Publish to PyPI if: inputs.dry_run != true @@ -284,7 +396,7 @@ jobs: git add datafog/__about__.py setup.py git commit -m "chore: bump version to $VERSION [skip ci]" || echo "No version changes to commit" - git push origin "$BRANCH" + git push origin "HEAD:$BRANCH" git tag -a "v$VERSION" -m "Release $VERSION" git push origin "v$VERSION" diff --git a/AGENTS.md b/AGENTS.md index 1a174106..13439692 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,9 +13,9 @@ ## Current Project Status -**Stable version: 4.8.0** +**Stable version: 4.9.0** -**Development version: 4.8.0** +**Development version: 4.9.0** **Next major target: 5.0.0** diff --git a/CHANGELOG.MD b/CHANGELOG.MD index b5327ede..576597a3 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,14 +1,13 @@ # ChangeLog -## [Unreleased] +## [4.9.0] #### Added - Experimental `datafog[rust]` detection with keyword-only `backend="rust"` - on scan/redact entry points; Python remains the default. The follow-up requires - `datafog-core>=0.4.0,<0.5` and capability contract 1. Core 0.4.0 is published - and registry-wheel validation passed; DataFog Python remains unreleased. - ML composition is unchanged. + on scan/redact entry points; Python remains the default. Requires + `datafog-core>=0.4.0,<0.5` and capability contract 1. The base installation + does not install or import Core. ML composition is unchanged. - Capability-driven entity and locale discovery, German locale activation, explicit UUID activation, and forward-compatible finding labels. Core's structured-only PERSON is rejected for explicit Rust text selection. @@ -42,7 +41,7 @@ - **Redaction token numbering now follows document order**: `redact()` previously assigned per-type counters while replacing spans right-to-left, - so with multiple entities of the same type the *last* occurrence received + so with multiple entities of the same type the _last_ occurrence received `_1` (two emails redacted as `[EMAIL_2] ... [EMAIL_1]`). Tokens for the `token` and `pseudonymize` strategies are now numbered left-to-right, so the first occurrence is always `_1`. This changes redacted output text for @@ -347,6 +346,7 @@ opt-in. No changes to the core library or its dependencies. #### Migration Guide For users upgrading from v4.1.1: + - All existing functionality remains unchanged - To use GLiNER: `pip install datafog[nlp-advanced]` - Smart cascading: `TextService(engine="smart")` for best balance diff --git a/README.md b/README.md index 799fd689..4fd354a3 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ scan/redact helpers, or guardrail helpers. model. - A Java runtime is required by PySpark. -The upcoming 4.9 release deprecates OCR and Spark with visible use-time +DataFog 4.9.0 deprecates OCR and Spark with visible use-time warnings; their APIs and extras will be removed in 5.0. They remain functional in 4.9. Users who need these features can stay on the final 4.x release. See the [4.9 migration guide](docs/migration-4.9.md) for the transition plan. @@ -261,9 +261,11 @@ The [4.9 migration guide](docs/migration-4.9.md) explains opt-in Rust detection, the native `datafog.v5` preview, and the revised 5.0 retirement schedule for `detect`/`process`, OCR, and Spark. The Python detector remains the default. -The unreleased Rust adapter requires Core `>=0.4.0,<0.5` and capability contract 1. -Core 0.4.0 is available on PyPI; install the development checkout with -`python -m pip install -e ".[rust]"` to evaluate this unreleased Python adapter. +The experimental Rust adapter in 4.9.0 requires Core `>=0.4.0,<0.5` and capability +contract 1. Upgrade with `python -m pip install --upgrade "datafog[rust]==4.9.0"` +to evaluate it. Base-only users can install `datafog==4.9.0` without Core; +installing the Rust extra does not change the default backend. See the +[4.9.0 release notes](RELEASE_NOTES_4.9.0.md). Entity labels, locales, and activation settings come from the installed Core, allowing compatible releases to add detectors without a Python update. German detection is opt-in through locale or entity selection; UUID is opt-in diff --git a/RELEASE_NOTES_4.9.0.md b/RELEASE_NOTES_4.9.0.md new file mode 100644 index 00000000..8e97ebd4 --- /dev/null +++ b/RELEASE_NOTES_4.9.0.md @@ -0,0 +1,91 @@ +# DataFog Python 4.9.0 + +4.9.0 bridges the existing Python API and DataFog Core. **Python detection remains +the default.** Existing imports, result classes, and legacy redaction strategies +continue to work. Rust detection and the native API preview are experimental, +explicit opt-ins. + +## Upgrade + +```bash +# Base package: no native dependency is automatically installed. +python -m pip install --upgrade "datafog==4.9.0" + +# Optional Rust backend and native API preview. +python -m pip install --upgrade "datafog[rust]==4.9.0" +``` + +The Rust extra requires `datafog-core>=0.4.0,<0.5` and capability contract `1`. +Installing it does not select Rust automatically. The `all` extra does not +include Rust; request `datafog[all,rust]==4.9.0` if both are needed. Lock your Core +version when detection output must be reproducible across installations. + +```python +import datafog + +result = datafog.redact("Contact jane@example.com", engine="regex", backend="rust") +assert result.redacted_text == "Contact [EMAIL_1]" +``` + +Rust is supported only with `engine="regex"`. Missing dependencies, unsupported +configurations, and native failures raise errors without silently falling back. +The CLI, `DataFog`, `TextService`, ML engines, and existing integrations retain +their current detection paths. + +## What changes + +- `datafog.compat.v4` exposes existing scan/redact APIs and result classes with + the same class identity as the established Python API. +- `datafog.v5` exposes native Core types and operations. Scanning returns + `list[Finding]`; native transformations return `TransformResult`, use Core's + strategies, and do not promise legacy numbered tokens or plaintext mappings. +- Rust detector labels, supported locales, and opt-in settings come from Core's + capability metadata. Core 0.4.0 supports the seven German detectors through + German locales or explicit entity selection. UUID remains opt-in. Core's + structured-only `PERSON` is rejected for explicit Rust text selection. +- Core 0.4.0 adds JWT, private-key, contextual US routing-number, and contextual + NPI detection. Python allowlists and legacy transformations remain in the + compatibility adapter. + +## Known differences + +Rust is not advertised as universally equivalent to the Python detector. On the +frozen 111-case baseline there are 77 exact matches, two reviewed detector +differences, one validation-message difference, and 31 cases outside backend +scope. Core rejects invalid-checksum cards and alphanumeric-embedded SSNs in the +reviewed differing cases. Unsupported locale validation uses Core capabilities. + +**Explicit NPI selection has a compatibility limitation:** when Core reports both +`PHONE` and `NPI` for the same span, legacy overlap handling keeps `PHONE` before +entity filtering. Consequently `entity_types=["NPI"]` can return no entities. +Default legacy redaction still protects that span as a phone. Native +`datafog.v5.scan()` retains NPI, and native transformation can select it. Choose +the native API when retaining NPI identity is required. + +The adapter resolves overlaps before entity filtering. A later compatible Core +release can introduce a finding that wins an overlap and suppresses a previously +selected label. The dependency range promises API compatibility, not identical +detection or selection results. For reproducible behavior, pin both packages: + +```bash +python -m pip install "datafog[rust]==4.9.0" "datafog-core==0.4.0" +``` + +Use native findings and native transformation entity selection when preserving +specific overlapping labels is required. + +## Deprecations for 5.0 + +`detect()` and `process()` still work in 4.9 but now warn of removal in 5.0. This +revises the earlier promise to retain them throughout 5.x. Migrate to scan/redact +and review transformation differences, particularly older placeholder and hash +formats. + +OCR, Donut, Tesseract, image services, Spark, and distributed helpers remain +functional in 4.9 with visible use-time warnings. Their removal is planned for +5.0. Users needing them can remain on the final 4.x release; this does not promise +indefinite maintenance or introduce successor packages. spaCy and GLiNER are not +removed by this release. + +See the [migration guide](https://github.com/DataFog/datafog-python/blob/v4.9.0/docs/migration-4.9.md) for API examples, compatibility +boundaries, parity evidence, and reproducible verification commands. diff --git a/datafog/__about__.py b/datafog/__about__.py index c9d2d193..454a2eda 100644 --- a/datafog/__about__.py +++ b/datafog/__about__.py @@ -1 +1 @@ -__version__ = "4.8.0a13" +__version__ = "4.9.0" diff --git a/docs/cli.rst b/docs/cli.rst index e27f4d72..b07a080a 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -8,7 +8,7 @@ The main entrypoint for the CLI is through the DataFog client file, defined in : We use Typer to build the CLI, with each command defined as a separate function. Core text commands such as ``scan-text``, ``redact-text``, ``replace-text``, -and ``hash-text`` are the primary CLI path. The unreleased 4.9 bridge leaves text commands on their +and ``hash-text`` are the primary CLI path. The 4.9.0 bridge leaves text commands on their existing Python detection paths; the opt-in Rust backend is a Python API option, not a new CLI flag. Install ``datafog[cli]`` for the command-line dependencies. @@ -26,7 +26,7 @@ commands. Spark support is also deprecated in 4.9 for removal in 5.0. Install ``datafog[distributed]`` when using ``SparkService`` during 4.x. Users needing OCR/Spark after the cutover can remain on the final 4.x release. See :doc:`optional-surfaces` and the -:download:`unreleased 4.9 migration guide `. +:download:`4.9 migration guide `. German locale support --------------------- diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 74234da6..11753c41 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -45,19 +45,20 @@ Optional extras are explicit: - ``pip install "datafog[all]"`` - You are developing or deliberately want every optional surface. -Unreleased 4.9 bridge -===================== +4.9.0 migration bridge +====================== -The following APIs are development previews, not a claim that 4.9 is published. -From a checkout containing the 4.9 implementation, install the explicit Rust -extra to evaluate them: +The native API preview and Rust backend are experimental additions in 4.9.0. +Upgrade the base package with ``python -m pip install --upgrade datafog==4.9.0`` +to keep using Python detection without a native dependency. To evaluate Rust, +install the optional extra explicitly: .. code-block:: bash - python -m pip install -e ".[rust]" + python -m pip install --upgrade "datafog[rust]==4.9.0" The extra requires ``datafog-core>=0.4.0,<0.5`` and capability contract 1. -Core 0.4.0 is available on PyPI; DataFog Python 4.9 remains unreleased. The extra +The extra does not change the default Python backend, and the existing ``all`` extra does not include Rust. To opt in: @@ -162,7 +163,7 @@ The CLI core path is text-first: datafog hash-text "Contact jane@example.com" datafog redact-text "Steuer-ID 12345678901" --locale de -Image commands are optional and scheduled for deprecation in 4.9 and removal +Image commands are optional, deprecated in 4.9, and scheduled for removal in 5.0. They remain functional in 4.9. Install ``datafog[ocr]`` for local OCR and ``datafog[web,ocr]`` when the CLI needs to download image inputs. diff --git a/docs/important-concepts.rst b/docs/important-concepts.rst index a1655593..239471bc 100644 --- a/docs/important-concepts.rst +++ b/docs/important-concepts.rst @@ -9,7 +9,7 @@ Overview Data Models ^^^^^^^^^^^ Existing models support legacy PII annotation and optional OCR analysis. -The unreleased 4.9 bridge retains them and adds ``datafog.compat.v4`` for the +The 4.9.0 bridge retains them and adds ``datafog.compat.v4`` for the legacy scan/redact result classes (``Entity``, ``ScanResult``, ``RedactResult``). ``datafog.v5`` separately previews native Core ``Finding`` and ``TransformResult`` objects; these have different fields and transformation semantics. See @@ -25,7 +25,7 @@ objects; these have different fields and transformation semantics. See Processors ^^^^^^^^^^^ Text processors remain available. OCR processors below are deprecated in the -unreleased 4.9 bridge and scheduled for removal in 5.0: +4.9.0 bridge and scheduled for removal in 5.0: * SpacyAnnotator Text annotation with spaCy diff --git a/docs/index.rst b/docs/index.rst index 44f94e63..df61a539 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,14 +11,13 @@ Start with :doc:`getting-started` if you want the shortest route from install to scanning text. The roadmap and historical planning pages remain available, but the live user docs are the first path for current text APIs. -4.9 development preview -======================= +4.9.0 migration bridge +====================== -.. note:: - - The 4.9 bridge described here is unreleased development work. These pages do - not announce a published 4.9 package. A normal PyPI install does not imply - availability of the new APIs below. +DataFog 4.9.0 is the migration bridge to the planned Rust-backed 5.0 API. +Upgrade the base package with ``python -m pip install --upgrade datafog==4.9.0``; +the native dependency remains optional. The ``rust`` extra requires +``datafog-core>=0.4.0,<0.5`` and capability contract version 1. 4.9 preserves existing imports, result classes, Python detection defaults, and redaction behavior. It adds explicit experimental Rust detection through @@ -26,7 +25,7 @@ redaction behavior. It adds explicit experimental Rust detection through Core schema preview. See :doc:`python-sdk` and the :download:`complete 4.9 migration guide `. -The planned 4.9 release deprecates ``detect()``/``process()``, OCR, and Spark for +DataFog 4.9.0 deprecates ``detect()``/``process()``, OCR, and Spark for removal in 5.0. The earlier promise to retain ``detect()``/``process()`` throughout 5.x is revised. OCR/Spark remain functional in 4.9; users needing them after the cutover can remain on the final 4.x release. See :doc:`optional-surfaces`. diff --git a/docs/migration-4.9.md b/docs/migration-4.9.md index dff739ae..3bf2b900 100644 --- a/docs/migration-4.9.md +++ b/docs/migration-4.9.md @@ -1,18 +1,21 @@ -# Migrating incrementally with DataFog 4.9 +# Migrating incrementally with DataFog 4.9.0 -> **Unreleased:** This guide describes the upcoming 4.9 bridge. Until it is -> published, install the development checkout with `python -m pip install -e -".[rust]"`. Core 0.4.0 is available on PyPI. A normal released DataFog Python -> install does not include this follow-up. - -4.9 is a bridge to the Rust-backed 5.0 API. The default Python detector, existing +4.9.0 is a bridge to the Rust-backed 5.0 API. The default Python detector, existing imports, result objects, and redaction strategies continue to work. The optional Rust backend and native API preview are experimental and explicitly selected. -## Opt into Rust detection +## Upgrade and opt into Rust detection + +Upgrade the base package while retaining Python detection and no native dependency: ```bash -pip install "datafog[rust]" +python -m pip install --upgrade "datafog==4.9.0" +``` + +Install the optional native dependency explicitly to evaluate Rust detection: + +```bash +python -m pip install --upgrade "datafog[rust]==4.9.0" ``` The extra requires `datafog-core>=0.4.0,<0.5` and capability contract version 1. @@ -204,3 +207,16 @@ for the short payload, 39.50 versus 7.58 microseconds for mixed PII, and 121.91 versus 5.33 milliseconds for the large sparse payload (Python versus Rust). Fresh-process import plus first scan was approximately 81 milliseconds for both. These are local measurements, not release performance guarantees. + +## Detector upgrades and overlapping labels + +Capability discovery allows new Core labels to pass through the adapter without +a Python inventory update; it does not guarantee unchanged selected results. +The legacy adapter resolves overlaps before applying `entity_types`. A new +detector can therefore suppress a previously selected label, leaving an explicit +selection empty. The NPI/PHONE case above is one concrete example. + +Pin `datafog-core==0.4.0` alongside `datafog==4.9.0` for reproducibility against +this release's validated detector set. Test detector upgrades against your own +selection policies. Use `datafog.v5.scan()` to retain native candidates and +native transformation entity selection when overlapping label identity matters. diff --git a/docs/python-sdk.rst b/docs/python-sdk.rst index e126c94d..2a8a2ddf 100644 --- a/docs/python-sdk.rst +++ b/docs/python-sdk.rst @@ -27,11 +27,11 @@ available for existing users. ``TextService(engine="regex")`` is the dependency-light service path; ``spacy``, ``gliner``, ``smart``, OCR, and Spark surfaces require their explicit extras. -4.9 compatibility and Core preview (unreleased) ------------------------------------------------ +4.9.0 compatibility and Core preview +------------------------------------ -These additions describe development work for 4.9; they do not indicate a -published release. Existing top-level ``scan``/``redact`` functions retain their +DataFog 4.9.0 provides an experimental Rust backend and native schema preview. +Existing top-level ``scan``/``redact`` functions retain their result shapes and use the Python backend by default. They delegate through the new facade, whose classes are the same objects as the established result types: @@ -44,9 +44,10 @@ continue to work at the top level in 4.9 but warn of removal in 5.0, revising th previous promise to retain them throughout 5.x. Moving from ``process`` to ``redact`` can change old placeholder and hash output; compare results explicitly. -The unreleased adapter requires ``datafog-core>=0.4.0,<0.5``. Core 0.4.0 is -available on PyPI. Install ``.[rust]`` from the development checkout to evaluate -Rust detection; DataFog Python 4.9 itself is not yet released: +The adapter requires ``datafog-core>=0.4.0,<0.5``. Install with +``python -m pip install --upgrade "datafog[rust]==4.9.0"`` to evaluate Rust. +A base install of ``datafog==4.9.0`` has no native dependency; installing the +extra does not change the default Python backend: .. code-block:: python @@ -155,7 +156,7 @@ OCR and Spark remain available as optional surfaces throughout 4.9: * Use ``datafog[distributed,nlp]`` plus an installed spaCy model for Spark PII UDF helpers. -The unreleased 4.9 bridge deprecates OCR and Spark for removal in 5.0. Use-time +DataFog 4.9.0 deprecates OCR and Spark for removal in 5.0. Use-time ``FutureWarning`` notices are visible under normal Python warning filters. Users needing these features can remain on the final 4.x release; this migration does not introduce successor packages or promise indefinite maintenance. See diff --git a/docs/roadmap.rst b/docs/roadmap.rst index a4aee1f8..3b7fd12c 100644 --- a/docs/roadmap.rst +++ b/docs/roadmap.rst @@ -4,7 +4,7 @@ Release Roadmap .. note:: - This earlier planning document is retained for context. The upcoming 4.9 + This earlier planning document is retained for context. The 4.9.0 bridge revises its compatibility commitments: ``detect``/``process``, OCR, and Spark are deprecated in 4.9 and removed in 5.0. The former promise to retain the shims through 5.x no longer applies. See the diff --git a/docs/v5-compatibility-matrix.rst b/docs/v5-compatibility-matrix.rst index ac29c378..b953729d 100644 --- a/docs/v5-compatibility-matrix.rst +++ b/docs/v5-compatibility-matrix.rst @@ -4,7 +4,7 @@ v5 Compatibility Matrix .. note:: - This earlier planning document is retained for context. The upcoming 4.9 + This earlier planning document is retained for context. The 4.9.0 bridge revises its compatibility commitments: ``detect``/``process``, OCR, and Spark are deprecated in 4.9 and removed in 5.0. The former promise to retain the shims through 5.x no longer applies. See the