From 21e9b8dcd61bb863252d8fb0955da05632293db9 Mon Sep 17 00:00:00 2001 From: Arda Tekin <2003arda2003@gmail.com> Date: Fri, 20 Nov 2026 00:59:28 +0200 Subject: [PATCH] feat: add tablassert publish-hf for Hugging Face dataset export --- CHANGELOG.md | 1 + README.md | 1 + docs/cli.md | 23 ++++++- docs/index.md | 2 +- docs/installation.md | 2 + llms.txt | 6 +- pyproject.toml | 3 + scripts/vulture_whitelist.py | 4 ++ src/tablassert/cli.py | 110 +++++++++++++++++++++++++++++++ src/tablassert/extras.py | 2 + tests/test_docs_cli_coverage.py | 2 + tests/test_extras.py | 3 +- tests/test_publish_hf.py | 111 ++++++++++++++++++++++++++++++++ uv.lock | 68 ++++++++++--------- 14 files changed, 300 insertions(+), 38 deletions(-) create mode 100644 tests/test_publish_hf.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a64043d..58af8aad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project are documented in this file. ### Added +- **`tablassert publish-hf` exports validated KGX outputs and RIG metadata to Hugging Face Hub.** Behind the opt-in `[hf]` extra (`huggingface-hub`), it gates on `validate_kgx`, generates a RIG-derived `README.md` dataset card with `nodes`/`edges` Parquet configs, attaches raw NDJSON and `.RIG.yaml`, and supports `--dry-run`. ([#269](https://github.com/SkyeAv/Tablassert/issues/269)) - **NDJSON (JSONL) is a fifth table source kind.** `kind: ndjson` scans newline-delimited JSON lazily with `pl.scan_ndjson` and renames its columns positionally to `column_1..` in first-seen key order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Ragged records scan missing keys as nulls instead of dropping rows; an empty file fails fast with a coded `ndjson-empty` error instead of polars' opaque `Cannot infer NDJSON types` compute error. ([#283](https://github.com/SkyeAv/Tablassert/pull/283), [#262](https://github.com/SkyeAv/Tablassert/issues/262)) - **Every pull request must carry a new changelog entry.** A stdlib-only `scripts/check_changelog.py` requires at least one new bullet under the `## Unreleased` heading of `CHANGELOG.md` relative to the pull request's merge base, failing with remediation guidance when whitespace, subheadings, or released-history edits are the only additions; a CI job runs it on every pull request including forks, release-bump and dependency-bump pull requests are exempt by their `chore(release):` / `chore(deps):` title or commit subject, documentation-only pull requests (every changed path is markdown or under `docs/`) are exempt via `--exempt-docs-only`, and a pre-push hook mirrors the same check locally. ([#284](https://github.com/SkyeAv/Tablassert/pull/284), [#282](https://github.com/SkyeAv/Tablassert/issues/282)) - **Arrow IPC (Feather v2) is a fourth table source kind.** `kind: ipc` scans a `.arrow` file lazily with `pl.scan_ipc` and renames its columns positionally to `column_1..` in schema order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Only IPC file layout is supported; a stream-layout file fails fast with a coded `ipc-stream-unsupported` error naming the conversion fix instead of polars' opaque `InvalidFooter` compute error. ([#281](https://github.com/SkyeAv/Tablassert/pull/281), [#264](https://github.com/SkyeAv/Tablassert/issues/264)) diff --git a/README.md b/README.md index da8bb8bd..73949793 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,7 @@ Python API to your project. Other extras are opt-in: | `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` | | `qc` | four-stage QC audit (exact match, fuzzy match, abbreviation expansion, and SapBERT embeddings) | `pip install "tablassert[qc]"` | | `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` | +| `hf` | Hugging Face Hub dataset export (`publish-hf`) | `pip install "tablassert[hf]"` | Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure names the missing package and the install command that fixes it. QC is opt-in at build time diff --git a/docs/cli.md b/docs/cli.md index f92989b8..339e25a4 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -2,7 +2,7 @@ Tablassert extracts knowledge assertions from tabular data into KGX NDJSON. The `tablassert` app requires the `[cli]` extra (`pip install "tablassert[cli]"` or `uv tool install "tablassert[cli]"`) and -exposes **five subcommands**: `build-fullmap`, `build-kg`, `quick-map`, `validate`, +exposes **six subcommands**: `build-fullmap`, `build-kg`, `publish-hf`, `quick-map`, `validate`, and `validate-kgx`, plus an app-level `--version` flag. Run `tablassert --help` (or ` --help`) for the live surface. @@ -12,6 +12,7 @@ for the live surface. | --- | --- | | [`build-fullmap`](#build-fullmap) | Build the embedded fullmap redb used for entity resolution | | [`build-kg`](#build-kg) | Build a KGX NDJSON knowledge graph from a YAML configuration | +| [`publish-hf`](#publish-hf) | Validate and publish KGX outputs and RIG metadata to Hugging Face Hub | | [`quick-map`](#quick-map) | Show what fullmap entity resolution does with one or more terms | | [`validate`](#validate) | Validate a graph or table configuration without executing it | | [`validate-kgx`](#validate-kgx) | Validate built KGX NDJSON against the Biolink Model and the Translator registry | @@ -242,6 +243,26 @@ tablassert validate graph.yaml --schema graph --- +## publish-hf + +Validate built KGX artifacts with `validate-kgx` and publish them (Parquet-converted `nodes` and +`edges` configs, raw NDJSONs, `.RIG.yaml`, and a RIG-derived `README.md` dataset card) to the +Hugging Face Hub (requires the `[hf]` extra). + +```bash +tablassert publish-hf graph.yaml --repo-id org/my-kg --dry-run +tablassert publish-hf -f graph.yaml -r org/my-kg +``` + +| Option | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `GRAPH-CONFIGURATION-FILE` (`--configuration-file`, `-f`) | Path | Yes | n/a | Graph YAML configuration file | +| `--repo-id`, `-r` | String | Yes | n/a | Target Hugging Face dataset repository ID (`org/dataset`) | +| `--private` | Flag | No | `false` | Create the dataset repository as private if it does not exist | +| `--dry-run` | Flag | No | `false` | Print the upload plan without contacting the Hugging Face Hub | + +--- + ## validate-kgx Use this to check that a completed build is actually Biolink-compliant and that the knowledge-source diff --git a/docs/index.md b/docs/index.md index ae64a6d9..a7ae9f86 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,7 +32,7 @@ expectations. ### How-to guides -- **[CLI Reference](cli.md)**: every command and flag: `build-kg`, `build-fullmap`, `validate`, +- **[CLI Reference](cli.md)**: every command and flag: `build-kg`, `build-fullmap`, `publish-hf`, `validate`, `validate-kgx`, `quick-map` - **[Fullmap](fullmap.md)**: obtain and build the entity-resolution database diff --git a/docs/installation.md b/docs/installation.md index e7f83b35..a921849d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -38,6 +38,7 @@ uv tool install "tablassert[cli]" # or: pip install "tablassert[cli]" | `aria2` | Bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `aria2==0.0.1b0` (imports as `aria2c`, bundles aria2c) | | `qc` | QC runtime (exact match, fuzzy match, abbreviation expansion, then a SapBERT embedding audit) | `scikit-learn`, `sentence-transformers` (`torch` + `numpy` arrive transitively; `rapidfuzz` is a core dependency) | | `log` | loguru-backed file/progress logging (rotation, enqueue) | `loguru` | +| `hf` | Hugging Face Hub dataset export (`publish-hf`) | `huggingface-hub` | Install any extra the same way: `uv tool install "tablassert[]"` or `pip install "tablassert[]"`, and combine them as `"tablassert[cli,qc]"`. Wheels ship for @@ -72,6 +73,7 @@ fails, not a failure report): | `tablassert` | `[cli]` | Before importing the command application, so a base Python-API install reports the exact install command instead of a bare module error | | `build-kg --qc` | `[qc]` | Before the build starts: the QC audit runs at the very end of the build, so a late failure would cost the entire entity-resolution pass | | `build-fullmap` | `[aria2]` | Before the first download, to pick the downloader: bundled aria2c when the extra is installed, Python downloader otherwise (announced on stderr and logged either way; never a missing-extra failure) | +| `publish-hf` | `[hf]` | Before validating and uploading KGX artifacts to the Hugging Face Hub | A partially installed extra names every package it is still missing, so installing them is one step rather than a retry loop. Library calls that reach an optional import directly (for example diff --git a/llms.txt b/llms.txt index 3091f560..5f5d7071 100644 --- a/llms.txt +++ b/llms.txt @@ -9,8 +9,8 @@ Current CLI behavior queries an embedded redb database whose primary file is `fu ## Quickstart - [README](README.md): high-level overview, install snippets, and one-command graph build. - [Documentation Site Home](docs/index.md): docs landing page and navigation context. -- [Installation Guide](docs/installation.md): three install paths (PyPI, GitHub main, development from source), platform and Rust-toolchain notes, and the missing-extra behavior. Optional extras: `cli`, `rt`, `aria2`, `qc`, and `log`. -- [CLI Reference](docs/cli.md): the five commands (`build-fullmap`, `build-kg`, `quick-map`, `validate`, and `validate-kgx`) plus the app-level `--version` flag. +- [Installation Guide](docs/installation.md): three install paths (PyPI, GitHub main, development from source), platform and Rust-toolchain notes, and the missing-extra behavior. Optional extras: `cli`, `rt`, `aria2`, `qc`, `log`, and `hf`. +- [CLI Reference](docs/cli.md): the six commands (`build-fullmap`, `build-kg`, `publish-hf`, `quick-map`, `validate`, and `validate-kgx`) plus the app-level `--version` flag. - [Tutorial](docs/tutorial.md): first end-to-end run from CSV input to KGX NDJSON output. ## YAML Authoring @@ -35,7 +35,7 @@ Current CLI behavior queries an embedded redb database whose primary file is `fu ## Contributor Development - [Contributing Guide](CONTRIBUTING.md): setup, style conventions, checks, and PR expectations. - [Development](docs/development.md): docs-site contributor setup and the local `uv run mkdocs build --strict` docs gate. -- [Project Metadata](pyproject.toml): dependencies, tooling config, and CLI entrypoint. Optional extras: `cli`, `rt`, `aria2`, `qc`, and `log`. +- [Project Metadata](pyproject.toml): dependencies, tooling config, and CLI entrypoint. Optional extras: `cli`, `rt`, `aria2`, `qc`, `log`, and `hf`. - [Tests](tests/): pytest coverage and fixtures. - [MkDocs Configuration](mkdocs.yml): docs navigation and published pages. - [GitHub Workflows](.github/workflows/): `ci.yml` runs lint, format, type, dead-code (vulture, deptry), test, rust (clippy, machete), and mkdocs checks when a release changes the project version or on manual dispatch; `docs.yml` builds and deploys MkDocs; `pipy.yml` checks the project version, builds sdist and wheel artifacts, and publishes to PyPI; `tag-version.yml` tags changed project versions; `paper.yml` tests the JOSS paper compile and uploads the PDF artifact; `security.yml` scans commits for leaked secrets with gitleaks and audits dependencies against advisory databases with pip-audit and cargo-audit for the Rust lockfile. diff --git a/pyproject.toml b/pyproject.toml index 7a4e2e90..afad3879 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,6 +98,9 @@ qc = [ log = [ "loguru>=0.7.3", ] +hf = [ + "huggingface-hub>=0.25.0", +] [dependency-groups] test = [ diff --git a/scripts/vulture_whitelist.py b/scripts/vulture_whitelist.py index 2d7b3485..16f5f248 100644 --- a/scripts/vulture_whitelist.py +++ b/scripts/vulture_whitelist.py @@ -20,6 +20,7 @@ from typing import Any +from tablassert.cli import publish_hf from tablassert.enums import ContentCategories, DataFormats, FillMethods, IngestCategories, ModelingCategories, ProvisionMechanisms, SourceStatuses from tablassert.models import ( RIGFilteredContent, @@ -105,3 +106,6 @@ RIGFutureModelingConsideration.consideration RIGSupportingDataSourceInfo.terms_of_use_info RIGProvenanceInfo.artifacts + +# cli.py: Cyclopts @APP.command callbacks registered dynamically at import time. +publish_hf diff --git a/src/tablassert/cli.py b/src/tablassert/cli.py index 2fc798b2..192bf32b 100644 --- a/src/tablassert/cli.py +++ b/src/tablassert/cli.py @@ -987,6 +987,116 @@ def validate( run(3, validate_pipeline, configuration_file) +def _hf_dataset_card(g: Any) -> str: + """Render a Hugging Face dataset card README.md from Graph and RIG metadata.""" + rig = getattr(g, "rig", None) + if rig is not None: + src = rig.source_info + terms = src.terms_of_use_info + license_line: str = terms.license_name or terms.terms_of_use_description or "See source terms of use" + contributions: str = "\n".join(f"- {c}" for c in rig.provenance_info.contributions) + description: str = src.description or rig.ingest_info.utility + infores_id: str = src.infores_id + else: + license_line = "See source terms of use" + contributions = "- Tablassert" + description = f"Knowledge graph {g.name} ({g.version}) built with Tablassert." + infores_id = getattr(g, "infores_id", "infores:tablassert") + return ( + "---\n" + "configs:\n" + " - config_name: nodes\n" + " data_files:\n" + " - split: train\n" + " path: nodes/*.parquet\n" + " - config_name: edges\n" + " data_files:\n" + " - split: train\n" + " path: edges/*.parquet\n" + "---\n\n" + f"# {g.name} ({g.version})\n\n" + f"{description}\n\n" + f"- **Infores:** `{infores_id}`\n" + f"- **Terms of use:** {license_line}\n\n" + "## Contributions\n\n" + f"{contributions}\n" + ) + + +@APP.command(name="publish-hf") +def publish_hf( + graph_configuration_file: Annotated[Path, cyclopts.Parameter(name=["--configuration-file", "-f"])], + repo_id: Annotated[str, cyclopts.Parameter(name=["--repo-id", "-r"])], + private: Annotated[bool, cyclopts.Parameter(name=["--private"], negative="")] = False, + dry_run: Annotated[bool, cyclopts.Parameter(name=["--dry-run"], negative="")] = False, +) -> None: + """Validate and publish built KGX outputs and RIG metadata as a Hugging Face dataset. + + Args: + graph_configuration_file: Path to the Graph YAML configuration file. + repo_id: Target Hugging Face dataset repository ID (e.g. ``org/my-kg``). + private: Create the dataset repository as private if it does not exist. + dry_run: Print the upload plan without contacting the Hugging Face Hub. + """ + from tempfile import TemporaryDirectory + + extras.require("hf", required_by="publish-hf") + from tablassert import biolink, ingests + from tablassert.models import Graph + + raw: object = ingests.from_yaml(graph_configuration_file) + g: Graph = Graph.model_validate(raw) + stem: str = f"{g.name}_{g.version}" + rig = getattr(g, "rig", None) + configured_dir: Path = g.artifact_path if hasattr(g, "artifact_path") else (rig.artifact_base_path if rig is not None else Path(".")) # pyright: ignore[reportAttributeAccessIssue] + base_dir: Path = configured_dir if (configured_dir / f"{stem}.nodes.ndjson").is_file() else Path(".") + nodes_path: Path = base_dir / f"{stem}.nodes.ndjson" + edges_path: Path = base_dir / f"{stem}.edges.ndjson" + rig_path: Path = base_dir / f"{stem}.RIG.yaml" + + if not nodes_path.is_file() or not edges_path.is_file(): + print(f"tablassert publish-hf: built KGX artifacts not found ({nodes_path}, {edges_path}).", file=sys.stderr) + raise SystemExit(1) + + report: dict[str, Any] = biolink.validate_kgx(nodes_path, edges_path) + if not report.get("ok", report["nodes"]["failures"] == 0 and report["edges"]["failures"] == 0): + print( + f"tablassert publish-hf: refusing to publish invalid KGX " + f"(nodes failures={report['nodes']['failures']}, edges failures={report['edges']['failures']}).", + file=sys.stderr, + ) + raise SystemExit(1) + + staged_files: list[str] = ["README.md", "nodes/data.parquet", "edges/data.parquet", nodes_path.name, edges_path.name] + if rig_path.is_file(): + staged_files.append(rig_path.name) + + if dry_run: + print(f"Dry run for Hugging Face dataset {repo_id} (private={private}):") + for item in staged_files: + print(f" - {item}") + return + + from huggingface_hub import HfApi # pyright: ignore[reportMissingImports] + + with TemporaryDirectory() as tmp: + stage: Path = Path(tmp) + (stage / "nodes").mkdir() + (stage / "edges").mkdir() + pl.read_ndjson(nodes_path).write_parquet(stage / "nodes" / "data.parquet") + pl.read_ndjson(edges_path).write_parquet(stage / "edges" / "data.parquet") + (stage / nodes_path.name).write_bytes(nodes_path.read_bytes()) + (stage / edges_path.name).write_bytes(edges_path.read_bytes()) + if rig_path.is_file(): + (stage / rig_path.name).write_bytes(rig_path.read_bytes()) + (stage / "README.md").write_text(_hf_dataset_card(g), encoding="utf-8") + + api = HfApi() + api.create_repo(repo_id=repo_id, repo_type="dataset", private=private, exist_ok=True) + api.upload_folder(folder_path=str(stage), repo_id=repo_id, repo_type="dataset") + print(f"Published {g.name} ({g.version}) to Hugging Face dataset {repo_id}.", file=sys.stderr) + + def _print_kgx_section(label: str, section: dict[str, Any], path: Path) -> None: """Print one ``validate-kgx`` per-file section (shared by the plain and ``--prune`` paths).""" if section["missing"]: diff --git a/src/tablassert/extras.py b/src/tablassert/extras.py index 07261825..2ad527ab 100644 --- a/src/tablassert/extras.py +++ b/src/tablassert/extras.py @@ -34,6 +34,7 @@ "aria2": {"aria2c": "aria2"}, "qc": {"sklearn": "scikit-learn", "sentence_transformers": "sentence-transformers"}, "log": {"loguru": "loguru"}, + "hf": {"huggingface_hub": "huggingface-hub"}, } # Import name -> owning extra, derived so the two can never drift apart. @@ -46,6 +47,7 @@ "aria2": "the bundled aria2c downloader (used automatically by build-fullmap when installed)", "qc": "the QC audit", "log": "file and progress logging", + "hf": "Hugging Face Hub dataset export", } diff --git a/tests/test_docs_cli_coverage.py b/tests/test_docs_cli_coverage.py index 6c802d6a..01d80476 100644 --- a/tests/test_docs_cli_coverage.py +++ b/tests/test_docs_cli_coverage.py @@ -23,6 +23,7 @@ COMMAND_NAME_DOCS: dict[str, tuple[str, ...]] = { "build-fullmap": ("cli.md", "fullmap.md"), "build-kg": ("cli.md",), + "publish-hf": ("cli.md",), "quick-map": ("cli.md", "fullmap.md"), "validate": ("cli.md",), "validate-kgx": ("cli.md",), @@ -30,6 +31,7 @@ COMMAND_FLAG_DOCS: dict[str, tuple[str, ...]] = { "build-fullmap": ("cli.md", "fullmap.md"), "build-kg": ("cli.md",), + "publish-hf": ("cli.md",), "quick-map": ("cli.md", "fullmap.md"), "validate": ("cli.md",), "validate-kgx": ("cli.md",), diff --git a/tests/test_extras.py b/tests/test_extras.py index 074c73be..9526fd41 100644 --- a/tests/test_extras.py +++ b/tests/test_extras.py @@ -40,7 +40,7 @@ def test_registry_covers_every_declared_extra() -> None: assert set(extras.EXTRA_PACKAGES) == declared - {"rt"} -@pytest.mark.parametrize("extra", ["cli", "qc", "aria2", "log"]) +@pytest.mark.parametrize("extra", ["cli", "qc", "aria2", "log", "hf"]) def test_registered_packages_are_actually_shipped_by_their_extra(extra: str) -> None: """Each import name maps to a distribution that its extra really installs. @@ -65,6 +65,7 @@ def test_module_lookup_is_derived_and_unambiguous() -> None: assert extras.EXTRA_FOR_MODULE["sklearn"] == "qc" assert extras.EXTRA_FOR_MODULE["sentence_transformers"] == "qc" assert extras.EXTRA_FOR_MODULE["loguru"] == "log" + assert extras.EXTRA_FOR_MODULE["huggingface_hub"] == "hf" assert len(extras.EXTRA_FOR_MODULE) == sum(len(packages) for packages in extras.EXTRA_PACKAGES.values()) diff --git a/tests/test_publish_hf.py b/tests/test_publish_hf.py new file mode 100644 index 00000000..087750d7 --- /dev/null +++ b/tests/test_publish_hf.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from tablassert import biolink, extras +from tablassert.cli import APP +from tablassert.errors import MissingExtraError + + +def _write_graph(tmp_path: Path, out_dir: Path) -> Path: + graph_path = tmp_path / "graph.yaml" + graph_path.write_text( + f"""name: demo-kg +version: 0.1.0 +tables: [] +fullmap: .fullmap +rig: + source_info: + infores_id: infores:demo-kg + description: Demo biomedical knowledge graph + terms_of_use_info: + license_name: CC BY 4.0 + data_access_locations: + - https://example.org/data + source_status: unknown + ingest_info: + utility: Demo utility + scope: Demo scope + provenance_info: + contributions: + - Demo Author - modeling + artifact_base_url: https://example.org/out + artifact_base_path: {out_dir} +""", + encoding="utf-8", + ) + return graph_path + + +def test_publish_hf_requires_extra_when_absent(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(extras, "find_spec", lambda name: None) + graph = _write_graph(tmp_path, tmp_path) + with pytest.raises(MissingExtraError, match="huggingface-hub"): + APP(["publish-hf", str(graph), "-r", "org/demo"], exit_on_error=False, result_action="return_value") + + +def test_publish_hf_refuses_invalid_kgx(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(extras, "require", lambda *a, **_kw: None) + out_dir = tmp_path / "out" + out_dir.mkdir() + (out_dir / "demo-kg_0.1.0.nodes.ndjson").write_text('{"id": "HGNC:1"}\n', encoding="utf-8") + (out_dir / "demo-kg_0.1.0.edges.ndjson").write_text('{"id": "e1"}\n', encoding="utf-8") + monkeypatch.setattr(biolink, "validate_kgx", lambda *a, **_kw: {"nodes": {"failures": 1}, "edges": {"failures": 0}}) + + graph = _write_graph(tmp_path, out_dir) + with pytest.raises(SystemExit) as exc: + APP(["publish-hf", str(graph), "-r", "org/demo"], exit_on_error=False, result_action="return_value") + assert exc.value.code == 1 + + +def test_publish_hf_dry_run_and_upload(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(extras, "require", lambda *a, **_kw: None) + out_dir = tmp_path / "out" + out_dir.mkdir() + (out_dir / "demo-kg_0.1.0.nodes.ndjson").write_text( + json.dumps({"id": "HGNC:1", "name": "A1BG", "category": ["biolink:Gene"]}) + "\n", encoding="utf-8" + ) + (out_dir / "demo-kg_0.1.0.edges.ndjson").write_text( + json.dumps({"id": "e1", "subject": "HGNC:1", "predicate": "biolink:related_to", "object": "HGNC:2"}) + "\n", encoding="utf-8" + ) + (out_dir / "demo-kg_0.1.0.RIG.yaml").write_text("name: demo\n", encoding="utf-8") + monkeypatch.setattr(biolink, "validate_kgx", lambda *a, **_kw: {"nodes": {"failures": 0}, "edges": {"failures": 0}}) + + graph = _write_graph(tmp_path, out_dir) + APP(["publish-hf", str(graph), "-r", "org/demo", "--dry-run"], exit_on_error=False, result_action="return_value") + captured = capsys.readouterr().out + assert "org/demo" in captured + assert "nodes/data.parquet" in captured + assert "demo-kg_0.1.0.RIG.yaml" in captured + + uploaded: list[dict[str, Any]] = [] + + class _FakeHfApi: + def create_repo(self, **kwargs: Any) -> None: + uploaded.append({"create": kwargs}) + + def upload_folder(self, **kwargs: Any) -> None: + readme = (Path(kwargs["folder_path"]) / "README.md").read_text(encoding="utf-8") + uploaded.append({"upload": kwargs, "readme": readme}) + + monkeypatch.setitem(sys.modules, "huggingface_hub", SimpleNamespace(HfApi=_FakeHfApi)) + APP(["publish-hf", str(graph), "-r", "org/demo", "--private"], exit_on_error=False, result_action="return_value") + assert len(uploaded) == 2 + assert uploaded[0]["create"]["private"] is True + assert "infores:demo-kg" in uploaded[1]["readme"] + assert "CC BY 4.0" in uploaded[1]["readme"] + + +def test_publish_hf_refuses_missing_artifacts(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(extras, "require", lambda *a, **_kw: None) + monkeypatch.chdir(tmp_path) + graph = _write_graph(tmp_path, tmp_path / "missing_out") + with pytest.raises(SystemExit) as exc: + APP(["publish-hf", str(graph), "-r", "org/demo"], exit_on_error=False, result_action="return_value") + assert exc.value.code == 1 diff --git a/uv.lock b/uv.lock index 9cb3ad61..bd314f14 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'win32'", @@ -51,8 +51,8 @@ name = "anyio" version = "4.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "idna" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "idna", marker = "sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } wheels = [ @@ -409,7 +409,7 @@ name = "cuda-bindings" version = "13.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/e0/a9/3a8241c6e19483ac1f1dcf5c10238205dcb8a6e9d0d4d4709240dff28ff4/cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d", size = 5730273, upload-time = "2026-03-11T00:12:37.18Z" }, @@ -442,40 +442,40 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", version = "13.4.1.3", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", version = "13.4.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cufft = [ - { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cufile = [ - { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cusolver = [ - { name = "nvidia-cublas", version = "13.4.1.3", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", version = "13.4.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -685,8 +685,8 @@ name = "httpcore2" version = "2.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h11" }, - { name = "truststore" }, + { name = "h11", marker = "sys_platform != 'emscripten'" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c0/61/afa9d6b7452f9eca73f73d45324eb0bb52f098fd5e6049bf6b8263be9b4e/httpcore2-2.16.0.tar.gz", hash = "sha256:c72cd0413563b17e1cd8354677dff887466f8ec2551679759e93126534428643", size = 69886, upload-time = "2026-10-21T07:09:45.31Z" } wheels = [ @@ -1216,7 +1216,7 @@ resolution-markers = [ "python_full_version < '3.12' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/d0/a0/5cd9d4b08a8320f7ca150ee0d66c97e98092922e338336cb4b62da7500c3/nvidia_cublas-13.4.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:4eb1472bf92b741e7cf24da7978dde251b58ddec2092d9f6a04f3623eb4e9a15", size = 513048232, upload-time = "2026-07-16T09:45:07.751Z" }, @@ -1264,8 +1264,8 @@ name = "nvidia-cudnn-cu13" version = "9.26.0.51" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", version = "13.1.0.3", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 's390x'" }, - { name = "nvidia-cublas", version = "13.4.1.3", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 's390x'" }, + { name = "nvidia-cublas", version = "13.1.0.3", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cublas", version = "13.4.1.3", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c5/1c/01e1dd3dec6ffe0baaff7e722b8946d867353d05a01301f128f1730fde0b/nvidia_cudnn_cu13-9.26.0.51-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8e5b4f5eccff721416f70ecf4353172fc4d5cdd8ca869fe6f983a4a5b0147b20", size = 595329497, upload-time = "2026-09-10T16:45:39.945Z" }, @@ -1277,7 +1277,7 @@ name = "nvidia-cufft" version = "12.2.0.57" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/07/8d/f650e6d527f9d22b5ded906c000eaebb037721c06d4ecd906df62f300979/nvidia_cufft-12.2.0.57-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ed438568ae1088ba2c25789d10205d4104b8b140b7bd942e39a787947a0fb74", size = 218244742, upload-time = "2026-07-16T09:47:08.76Z" }, @@ -1298,9 +1298,9 @@ name = "nvidia-cusolver" version = "12.2.0.11" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", version = "13.4.1.3", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", version = "13.4.1.3", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cusparse", marker = "platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/60/df/9e6b603f1d64040ce3e076e9bce1b78e2d588d0aabc9783f351cecd861ab/nvidia_cusolver-12.2.0.11-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8be097fca28de6be0990ab216ee9b804dd51077e1d1d83acf6c9850650bdc64d", size = 250383075, upload-time = "2026-07-16T09:50:16.355Z" }, @@ -1312,7 +1312,7 @@ name = "nvidia-cusparse" version = "12.7.10.12" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/60/35/abbd278fb6d039fcfb5175efa10a21007a11878bec59920d84854b47d766/nvidia_cusparse-12.7.10.12-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:597eff2cd83e3ab1d1e3a0dce9c9c879d77cf64ec2659bfdb5af6a5d02e7aaa9", size = 169043093, upload-time = "2026-07-16T09:51:37.058Z" }, @@ -1351,7 +1351,7 @@ name = "nvidia-nvshmem-cu13" version = "3.7.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-cccl" }, + { name = "nvidia-cuda-cccl", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/81/11/1767ef120299972822a43aceffbc81ac813aad2089d817e36e4e12aa4c9c/nvidia_nvshmem_cu13-3.7.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0e748308b9e0a8a8d81859353f3e6c98fbe85a85f652a915a2ba60149a0bd124", size = 135174619, upload-time = "2026-07-17T19:24:15.404Z" }, @@ -2601,6 +2601,9 @@ cli = [ { name = "cyclopts" }, { name = "rich" }, ] +hf = [ + { name = "huggingface-hub" }, +] log = [ { name = "loguru" }, ] @@ -2658,6 +2661,7 @@ requires-dist = [ { name = "biolink-model", specifier = ">=4.4.6" }, { name = "cyclopts", marker = "extra == 'cli'", specifier = ">=1.0.0" }, { name = "fastexcel", specifier = ">=0.21.0" }, + { name = "huggingface-hub", marker = "extra == 'hf'", specifier = ">=0.25.0" }, { name = "linkml-runtime", specifier = ">=1.11" }, { name = "loguru", marker = "extra == 'log'", specifier = ">=0.7.3" }, { name = "polars", specifier = ">=2.0.0rc2" }, @@ -2670,7 +2674,7 @@ requires-dist = [ { name = "scikit-learn", marker = "extra == 'qc'", specifier = ">=1.8.0" }, { name = "sentence-transformers", marker = "extra == 'qc'", specifier = ">=5.3.0" }, ] -provides-extras = ["cli", "rt", "aria2", "qc", "log"] +provides-extras = ["cli", "rt", "aria2", "qc", "log", "hf"] [package.metadata.requires-dev] ci = [