From 5d4cd3f96a14deee4e861ceb12a11f4783728c53 Mon Sep 17 00:00:00 2001 From: Arda Tekin <2003arda2003@gmail.com> Date: Fri, 20 Nov 2026 00:23:39 +0200 Subject: [PATCH] feat: add tablassert draft command for graph config generation --- CHANGELOG.md | 1 + docs/cli.md | 25 ++++++- docs/index.md | 2 +- llms.txt | 2 +- scripts/vulture_whitelist.py | 4 ++ src/tablassert/cli.py | 74 +++++++++++++++++++++ tests/test_docs_cli_coverage.py | 2 + tests/test_draft.py | 113 ++++++++++++++++++++++++++++++++ 8 files changed, 220 insertions(+), 3 deletions(-) create mode 100644 tests/test_draft.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b13f2edd..a697e71f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project are documented in this file. ### Added +- **`tablassert draft` generates a schema-valid draft graph configuration from table configs alone.** Given one or more table YAML paths, it derives `name` from the first table stem, sets `version: 0.1.0` and `fullmap: .fullmap`, populates honest `DRAFT` RIG placeholders (`infores:tablassert-draft`), validates the result against `Graph`, and writes YAML via `ingests.to_yaml` (refusing to overwrite without `--force`). ([#270](https://github.com/SkyeAv/Tablassert/issues/270)) - **NDJSON (JSONL) is a fifth table source kind.** `kind: ndjson` scans newline-delimited JSON lazily with `pl.scan_ndjson` and renames its columns positionally to `column_1..` in first-seen key order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Ragged records scan missing keys as nulls instead of dropping rows; an empty file fails fast with a coded `ndjson-empty` error instead of polars' opaque `Cannot infer NDJSON types` compute error. ([#283](https://github.com/SkyeAv/Tablassert/pull/283), [#262](https://github.com/SkyeAv/Tablassert/issues/262)) - **Every pull request must carry a new changelog entry.** A stdlib-only `scripts/check_changelog.py` requires at least one new bullet under the `## Unreleased` heading of `CHANGELOG.md` relative to the pull request's merge base, failing with remediation guidance when whitespace, subheadings, or released-history edits are the only additions; a CI job runs it on every pull request including forks, release-bump and dependency-bump pull requests are exempt by their `chore(release):` / `chore(deps):` title or commit subject, documentation-only pull requests (every changed path is markdown or under `docs/`) are exempt via `--exempt-docs-only`, and a pre-push hook mirrors the same check locally. ([#284](https://github.com/SkyeAv/Tablassert/pull/284), [#282](https://github.com/SkyeAv/Tablassert/issues/282)) - **Arrow IPC (Feather v2) is a fourth table source kind.** `kind: ipc` scans a `.arrow` file lazily with `pl.scan_ipc` and renames its columns positionally to `column_1..` in schema order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Only IPC file layout is supported; a stream-layout file fails fast with a coded `ipc-stream-unsupported` error naming the conversion fix instead of polars' opaque `InvalidFooter` compute error. ([#281](https://github.com/SkyeAv/Tablassert/pull/281), [#264](https://github.com/SkyeAv/Tablassert/issues/264)) diff --git a/docs/cli.md b/docs/cli.md index f92989b8..4eeedc79 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -2,7 +2,7 @@ Tablassert extracts knowledge assertions from tabular data into KGX NDJSON. The `tablassert` app requires the `[cli]` extra (`pip install "tablassert[cli]"` or `uv tool install "tablassert[cli]"`) and -exposes **five subcommands**: `build-fullmap`, `build-kg`, `quick-map`, `validate`, +exposes **six subcommands**: `build-fullmap`, `build-kg`, `draft`, `quick-map`, `validate`, and `validate-kgx`, plus an app-level `--version` flag. Run `tablassert --help` (or ` --help`) for the live surface. @@ -12,6 +12,7 @@ for the live surface. | --- | --- | | [`build-fullmap`](#build-fullmap) | Build the embedded fullmap redb used for entity resolution | | [`build-kg`](#build-kg) | Build a KGX NDJSON knowledge graph from a YAML configuration | +| [`draft`](#draft) | Generate a schema-valid draft graph config from one or more table configs | | [`quick-map`](#quick-map) | Show what fullmap entity resolution does with one or more terms | | [`validate`](#validate) | Validate a graph or table configuration without executing it | | [`validate-kgx`](#validate-kgx) | Validate built KGX NDJSON against the Biolink Model and the Translator registry | @@ -242,6 +243,28 @@ tablassert validate graph.yaml --schema graph --- +## draft + +Generate a schema-valid draft graph configuration from one or more existing table YAML files, with +explicit `DRAFT` placeholders for RIG metadata so `tablassert validate --schema graph` passes +immediately. + +```bash +tablassert draft TABLES... +tablassert draft table.yaml -o graph.yaml +``` + +| Option | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `TABLES` | Path (1+) | Yes | n/a | One or more existing table YAML configuration files | +| `--output`, `-o` | Path | No | `./graph.yaml` | Destination path for the generated draft graph config | +| `--force`, `-f` | Flag | No | `false` | Overwrite `--output` if it already exists | + +Refuses to overwrite an existing file unless `--force` is passed. Replace all `DRAFT` placeholders +before publishing. + +--- + ## validate-kgx Use this to check that a completed build is actually Biolink-compliant and that the knowledge-source diff --git a/docs/index.md b/docs/index.md index ae64a6d9..0252e7c3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,7 +32,7 @@ expectations. ### How-to guides -- **[CLI Reference](cli.md)**: every command and flag: `build-kg`, `build-fullmap`, `validate`, +- **[CLI Reference](cli.md)**: every command and flag: `build-kg`, `build-fullmap`, `draft`, `validate`, `validate-kgx`, `quick-map` - **[Fullmap](fullmap.md)**: obtain and build the entity-resolution database diff --git a/llms.txt b/llms.txt index dd93a4b5..3cd38144 100644 --- a/llms.txt +++ b/llms.txt @@ -10,7 +10,7 @@ Current CLI behavior queries an embedded redb database whose primary file is `fu - [README](README.md): high-level overview, install snippets, and one-command graph build. - [Documentation Site Home](docs/index.md): docs landing page and navigation context. - [Installation Guide](docs/installation.md): three install paths (PyPI, GitHub main, development from source), platform and Rust-toolchain notes, and the missing-extra behavior. Optional extras: `cli`, `rt`, `aria2`, `qc`, and `log`. -- [CLI Reference](docs/cli.md): the five commands (`build-fullmap`, `build-kg`, `quick-map`, `validate`, and `validate-kgx`) plus the app-level `--version` flag. +- [CLI Reference](docs/cli.md): the six commands (`build-fullmap`, `build-kg`, `draft`, `quick-map`, `validate`, and `validate-kgx`) plus the app-level `--version` flag. - [Tutorial](docs/tutorial.md): first end-to-end run from CSV input to KGX NDJSON output. ## YAML Authoring diff --git a/scripts/vulture_whitelist.py b/scripts/vulture_whitelist.py index 2d7b3485..9d2cd62b 100644 --- a/scripts/vulture_whitelist.py +++ b/scripts/vulture_whitelist.py @@ -20,6 +20,7 @@ from typing import Any +from tablassert.cli import draft from tablassert.enums import ContentCategories, DataFormats, FillMethods, IngestCategories, ModelingCategories, ProvisionMechanisms, SourceStatuses from tablassert.models import ( RIGFilteredContent, @@ -105,3 +106,6 @@ RIGFutureModelingConsideration.consideration RIGSupportingDataSourceInfo.terms_of_use_info RIGProvenanceInfo.artifacts + +# cli.py: Cyclopts @APP.command callbacks registered dynamically at import time. +draft diff --git a/src/tablassert/cli.py b/src/tablassert/cli.py index 2fc798b2..cc8922aa 100644 --- a/src/tablassert/cli.py +++ b/src/tablassert/cli.py @@ -987,6 +987,80 @@ def validate( run(3, validate_pipeline, configuration_file) +@APP.command(name="draft") +def draft( + tables: Annotated[list[Path], cyclopts.Parameter(allow_leading_hyphen=False, negative="")], + output: Annotated[Path, cyclopts.Parameter(name=["--output", "-o"])] = Path("graph.yaml"), + force: Annotated[bool, cyclopts.Parameter(name=["--force", "-f"], negative="")] = False, +) -> None: + """Generate a draft graph configuration from one or more table config files. + + Emits a schema-valid ``graph.yaml`` with honest ``DRAFT`` placeholders for RIG + provenance and metadata so you can run ``tablassert validate --schema graph`` + immediately and fill in the marked fields before publishing. + + Args: + tables: One or more existing table YAML configuration files. + output: Path where the draft graph YAML will be written (default: ``graph.yaml``). + force: Overwrite ``output`` if it already exists. + """ + from tablassert import ingests + from tablassert.enums import IngestCategories, SourceStatuses + from tablassert.models import Graph + + for table in tables: + if not table.is_file(): + print(f"tablassert draft: table config not found: {table}", file=sys.stderr) + raise SystemExit(1) + + if output.is_dir(): + print(f"tablassert draft: output path {output} is a directory.", file=sys.stderr) + raise SystemExit(1) + + resolved_output: Path = output.resolve() + if resolved_output in {table.resolve() for table in tables}: + print(f"tablassert draft: refusing to overwrite input table {output}.", file=sys.stderr) + raise SystemExit(1) + + if output.exists() and not force: + print(f"tablassert draft: {output} already exists (use --force to overwrite).", file=sys.stderr) + raise SystemExit(1) + + graph_name: str = tables[0].stem or "draft-graph" + draft_data: dict[str, Any] = { + "name": graph_name, + "version": "0.1.0", + "tables": [str(table) for table in tables], + "fullmap": ".fullmap", + "rig": { + "source_info": { + "infores_id": "infores:tablassert-draft", + "name": f"DRAFT: {graph_name}", + "description": "DRAFT: describe the upstream source, its scope, and how its knowledge is produced.", + "terms_of_use_info": {"terms_of_use_description": "DRAFT: assess upstream source terms of use or license before publishing."}, + "data_access_locations": ["https://example.org/DRAFT-replace-with-source-url"], + "source_status": SourceStatuses.UNKNOWN.value, + }, + "ingest_info": { + "ingest_categories": [IngestCategories.TRANSLATOR_KNOWLEDGE_CREATOR.value], + "utility": "DRAFT: describe why this source was ingested and its utility for Translator use cases.", + "scope": "DRAFT: describe what knowledge is included and excluded in this ingest.", + }, + "provenance_info": {"contributions": ["DRAFT: Your Name - code author, data modeling"]}, + "artifact_base_url": "https://example.org/DRAFT-replace-with-artifact-url", + "artifact_base_path": "./output", + }, + } + # Validate through the live Graph model so the emitted draft is schema-valid by construction. + Graph.model_validate(draft_data) + + output.parent.mkdir(parents=True, exist_ok=True) + ingests.to_yaml(output, draft_data) + header: str = "# DRAFT graph configuration generated by `tablassert draft`.\n# Review and replace every DRAFT placeholder before publishing.\n" + output.write_text(header + output.read_text(encoding="utf-8"), encoding="utf-8") + print(f"Wrote draft graph config to {output}; replace DRAFT placeholders before publishing.", file=sys.stderr) + + def _print_kgx_section(label: str, section: dict[str, Any], path: Path) -> None: """Print one ``validate-kgx`` per-file section (shared by the plain and ``--prune`` paths).""" if section["missing"]: diff --git a/tests/test_docs_cli_coverage.py b/tests/test_docs_cli_coverage.py index 6c802d6a..64f6da14 100644 --- a/tests/test_docs_cli_coverage.py +++ b/tests/test_docs_cli_coverage.py @@ -23,6 +23,7 @@ COMMAND_NAME_DOCS: dict[str, tuple[str, ...]] = { "build-fullmap": ("cli.md", "fullmap.md"), "build-kg": ("cli.md",), + "draft": ("cli.md",), "quick-map": ("cli.md", "fullmap.md"), "validate": ("cli.md",), "validate-kgx": ("cli.md",), @@ -30,6 +31,7 @@ COMMAND_FLAG_DOCS: dict[str, tuple[str, ...]] = { "build-fullmap": ("cli.md", "fullmap.md"), "build-kg": ("cli.md",), + "draft": ("cli.md",), "quick-map": ("cli.md", "fullmap.md"), "validate": ("cli.md",), "validate-kgx": ("cli.md",), diff --git a/tests/test_draft.py b/tests/test_draft.py new file mode 100644 index 00000000..600b810c --- /dev/null +++ b/tests/test_draft.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tablassert import ingests +from tablassert.cli import APP +from tablassert.models import Graph + + +def _write_minimal_table(path: Path) -> Path: + path.write_text( + """template: + source: + kind: text + local: data.tsv + delimiter: "\\t" + url: + - https://example.org/data.tsv + statement: + subject: + method: column + encoding: A + predicate: related_to + object: + method: column + encoding: B + provenance: + repo: PMID + publication: "12345678" +sections: + - {} +""", + encoding="utf-8", + ) + return path + + +def test_draft_generates_schema_valid_graph_config(tmp_path: Path) -> None: + table = _write_minimal_table(tmp_path / "my-table.yaml") + out = tmp_path / "graph.yaml" + + APP(["draft", str(table), "-o", str(out)], exit_on_error=False, result_action="return_value") + + assert out.is_file() + raw = ingests.from_yaml(out) + assert isinstance(raw, dict) + graph = Graph.model_validate(raw) + assert graph.name == "my-table" + assert graph.version == "0.1.0" + assert graph.tables == [table] + assert graph.fullmap == Path(".fullmap") + assert graph.rig is not None + assert graph.rig.source_info.infores_id == "infores:tablassert-draft" + APP(["validate", str(out), "--schema", "graph"], exit_on_error=False, result_action="return_value") + + +def test_draft_supports_multiple_tables(tmp_path: Path) -> None: + t1 = _write_minimal_table(tmp_path / "first.yaml") + t2 = _write_minimal_table(tmp_path / "second.yaml") + out = tmp_path / "graph.yaml" + + APP(["draft", str(t1), str(t2), "--output", str(out)], exit_on_error=False, result_action="return_value") + + raw = ingests.from_yaml(out) + assert isinstance(raw, dict) + graph = Graph.model_validate(raw) + assert graph.name == "first" + assert graph.tables == [t1, t2] + + +def test_draft_refuses_overwrite_without_force(tmp_path: Path) -> None: + table = _write_minimal_table(tmp_path / "table.yaml") + out = tmp_path / "graph.yaml" + out.write_text("existing: true\n", encoding="utf-8") + + with pytest.raises(SystemExit) as exc: + APP(["draft", str(table), "-o", str(out)], exit_on_error=False, result_action="return_value") + assert exc.value.code == 1 + assert out.read_text(encoding="utf-8") == "existing: true\n" + + APP(["draft", str(table), "-o", str(out), "--force"], exit_on_error=False, result_action="return_value") + raw = ingests.from_yaml(out) + assert isinstance(raw, dict) + assert Graph.model_validate(raw).name == "table" + + +def test_draft_fails_on_missing_table(tmp_path: Path) -> None: + missing = tmp_path / "does-not-exist.yaml" + out = tmp_path / "graph.yaml" + + with pytest.raises(SystemExit) as exc: + APP(["draft", str(missing), "-o", str(out)], exit_on_error=False, result_action="return_value") + assert exc.value.code == 1 + assert not out.exists() + + +def test_draft_refuses_to_overwrite_input_table_or_directory(tmp_path: Path) -> None: + table = _write_minimal_table(tmp_path / "table.yaml") + original = table.read_text(encoding="utf-8") + + with pytest.raises(SystemExit) as exc: + APP(["draft", str(table), "-o", str(table), "--force"], exit_on_error=False, result_action="return_value") + assert exc.value.code == 1 + assert table.read_text(encoding="utf-8") == original + + out_dir = tmp_path / "out_dir" + out_dir.mkdir() + with pytest.raises(SystemExit) as exc_dir: + APP(["draft", str(table), "-o", str(out_dir), "--force"], exit_on_error=False, result_action="return_value") + assert exc_dir.value.code == 1 + assert out_dir.is_dir()