From 3b78d30696e40f2ebd7ac96822591a91257b4c43 Mon Sep 17 00:00:00 2001 From: Arda Tekin <2003arda2003@gmail.com> Date: Fri, 20 Nov 2026 01:01:10 +0200 Subject: [PATCH] feat: inherit table templates from pinned GitHub YAML files --- CHANGELOG.md | 1 + src/tablassert/errors.py | 1 + src/tablassert/ingests.py | 93 ++++++++++++++++++++++++----- tests/test_github_templates.py | 105 +++++++++++++++++++++++++++++++++ 4 files changed, 185 insertions(+), 15 deletions(-) create mode 100644 tests/test_github_templates.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b13f2edd..47f016e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project are documented in this file. ### Added +- **Table configs can inherit `template:` definitions from pinned GitHub YAML files.** Setting `template.from: github://@` (with optional `select:`) fetches and caches the remote YAML under `.tablassert/templates/`, parses it with `CSafeLoader`, and deep-merges it under the local template via `fastmerge` so offline cached builds match inlined equivalents. ([#277](https://github.com/SkyeAv/Tablassert/issues/277)) - **NDJSON (JSONL) is a fifth table source kind.** `kind: ndjson` scans newline-delimited JSON lazily with `pl.scan_ndjson` and renames its columns positionally to `column_1..` in first-seen key order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Ragged records scan missing keys as nulls instead of dropping rows; an empty file fails fast with a coded `ndjson-empty` error instead of polars' opaque `Cannot infer NDJSON types` compute error. ([#283](https://github.com/SkyeAv/Tablassert/pull/283), [#262](https://github.com/SkyeAv/Tablassert/issues/262)) - **Every pull request must carry a new changelog entry.** A stdlib-only `scripts/check_changelog.py` requires at least one new bullet under the `## Unreleased` heading of `CHANGELOG.md` relative to the pull request's merge base, failing with remediation guidance when whitespace, subheadings, or released-history edits are the only additions; a CI job runs it on every pull request including forks, release-bump and dependency-bump pull requests are exempt by their `chore(release):` / `chore(deps):` title or commit subject, documentation-only pull requests (every changed path is markdown or under `docs/`) are exempt via `--exempt-docs-only`, and a pre-push hook mirrors the same check locally. ([#284](https://github.com/SkyeAv/Tablassert/pull/284), [#282](https://github.com/SkyeAv/Tablassert/issues/282)) - **Arrow IPC (Feather v2) is a fourth table source kind.** `kind: ipc` scans a `.arrow` file lazily with `pl.scan_ipc` and renames its columns positionally to `column_1..` in schema order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Only IPC file layout is supported; a stream-layout file fails fast with a coded `ipc-stream-unsupported` error naming the conversion fix instead of polars' opaque `InvalidFooter` compute error. ([#281](https://github.com/SkyeAv/Tablassert/pull/281), [#264](https://github.com/SkyeAv/Tablassert/issues/264)) diff --git a/src/tablassert/errors.py b/src/tablassert/errors.py index 978659dd..4788a762 100644 --- a/src/tablassert/errors.py +++ b/src/tablassert/errors.py @@ -90,6 +90,7 @@ def redact_secrets(text: str, secrets: Sequence[str] = ()) -> str: "rig-bad-infores", "rig-bad-artifact-url", "rig-bad-access-location", + "template-bad-github-ref", "rig-terms-empty", "rig-legacy-keys", "rig-validation-failed", diff --git a/src/tablassert/ingests.py b/src/tablassert/ingests.py index ce714021..e9ea449e 100644 --- a/src/tablassert/ingests.py +++ b/src/tablassert/ingests.py @@ -1,12 +1,18 @@ from __future__ import annotations +import hashlib +import re from copy import deepcopy from pathlib import Path from typing import Any +from urllib.error import URLError +from urllib.request import Request, urlopen import yaml from yaml import CSafeLoader +from tablassert.errors import TablassertError + def fastmerge(a: list[Any] | dict[str, Any], b: list[Any] | dict[str, Any]) -> Any: """Recursively merge ``b`` into ``a`` in place and return the result. @@ -82,27 +88,84 @@ def to_yaml(p: Path, data: object) -> None: yaml.dump(data, f, Dumper=_IndentedSafeDumper, sort_keys=False, default_flow_style=False, allow_unicode=True, width=120) -def to_sections(instructions: dict[str, Any], table: Path) -> list[dict[str, Any]]: +_GITHUB_SLUG_RE: re.Pattern[str] = re.compile(r"^[A-Za-z0-9_.-]+$") + + +def parse_github_ref(spec: str) -> tuple[str, str, str, str]: + """Parse and validate a pinned ``github://@`` template reference.""" + if not spec.startswith("github:") or "@" not in spec: + raise TablassertError(f"Remote template `from` must use `github://@`, got {spec!r}.", code="template-bad-github-ref") + body, ref = spec.removeprefix("github:").rsplit("@", 1) + parts: list[str] = body.split("/", 2) + if len(parts) != 3 or not all(parts) or not ref: + raise TablassertError(f"Remote template `from` must use `github://@`, got {spec!r}.", code="template-bad-github-ref") + owner, repo, path = parts + if not _GITHUB_SLUG_RE.match(owner) or owner in (".", "..") or not _GITHUB_SLUG_RE.match(repo) or repo in (".", ".."): + raise TablassertError(f"Invalid GitHub owner/repo in template ref {spec!r}.", code="template-bad-github-ref") + path_segments: list[str] = path.split("/") + if any(seg in ("", ".", "..") for seg in path_segments) or any(c in path for c in ("?", "#", "\\")) or any(c.isspace() for c in path): + raise TablassertError(f"Invalid file path in GitHub template ref {spec!r}.", code="template-bad-github-ref") + if ".." in ref or ref.startswith("/") or ref.endswith("/") or any(c in ref for c in ("?", "#", "\\")) or any(c.isspace() for c in ref): + raise TablassertError(f"Invalid git ref in GitHub template ref {spec!r}.", code="template-bad-github-ref") + return owner, repo, path, ref + + +def fetch_github_template(spec: str, *, cache_dir: Path | None = None) -> tuple[dict[str, Any], str]: + """Fetch (or read from local cache) a GitHub-hosted YAML template using CSafeLoader.""" + owner, repo, path, ref = parse_github_ref(spec) + cache_root: Path = cache_dir or Path(".tablassert/templates") + key: str = hashlib.sha256(f"{owner}/{repo}/{ref}/{path}".encode()).hexdigest() + cached_file: Path = cache_root / f"{key}.yaml" + + from_cache: bool = cached_file.is_file() + if from_cache: + raw_bytes: bytes = cached_file.read_bytes() + else: + url: str = f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}" + req = Request(url, headers={"User-Agent": "tablassert"}) + try: + with urlopen(req, timeout=30) as resp: + raw_bytes = resp.read() + except (URLError, OSError) as exc: + raise TablassertError(f"Failed to fetch remote template {spec!r}: {exc}", code="template-bad-github-ref") from exc + + try: + parsed: object = yaml.load(raw_bytes.decode("utf-8"), Loader=CSafeLoader) + except (UnicodeDecodeError, yaml.YAMLError) as exc: + raise TablassertError(f"Remote template {spec!r} failed YAML parsing: {exc}", code="template-bad-github-ref") from exc + if not isinstance(parsed, dict): + raise TablassertError(f"Remote template {spec!r} did not parse as a YAML mapping.", code="template-bad-github-ref") + + if not from_cache: + cache_root.mkdir(parents=True, exist_ok=True) + cached_file.write_bytes(raw_bytes) + digest: str = hashlib.sha256(raw_bytes).hexdigest() + return parsed, digest + + +def to_sections(instructions: dict[str, Any], table: Path, *, cache_dir: Path | None = None) -> list[dict[str, Any]]: """Expand a table config dict into a list of section dicts. Each section is the deep merge of the table-level ``template`` over a - per-section entry. The originating ``table`` path is stamped into each - section as ``config``, and the section's 1-based position in the file - (document order) as ``section_index`` so provenance can name the inlined - Study after its origin (" SECTION "). - - Args: - instructions: Parsed YAML config with optional ``template`` and - ``sections`` keys. - table: Path of the source table file (recorded on each section). - - Returns: - One merged dict per section. + per-section entry. If ``template.from`` references a pinned GitHub YAML + (``github:owner/repo/path.yaml@ref``), it is fetched/cached and merged under + the local ``template``. """ - template: dict[str, Any] = instructions.get("template", {}) + template: dict[str, Any] = deepcopy(instructions.get("template", {})) + remote_ref: object = template.pop("from", None) + select_key: str = str(template.pop("select", "template")) + if remote_ref is not None: + if not isinstance(remote_ref, str): + raise TablassertError(f"Remote template `from` must be a string, got {type(remote_ref).__name__}.", code="template-bad-github-ref") + remote_doc, _remote_digest = fetch_github_template(remote_ref, cache_dir=cache_dir) + selected: object = remote_doc.get(select_key) + if not isinstance(selected, dict): + raise TablassertError(f"Remote template {remote_ref!r} has no mapping under key {select_key!r}.", code="template-bad-github-ref") + template = fastmerge(deepcopy(selected), template) + template["config"] = table sections: list[dict[str, Any]] = instructions.get("sections", [{}]) - merged: list[dict[str, Any]] = [fastmerge(deepcopy(template), x) for x in sections] + merged: list[dict[str, Any]] = [fastmerge(deepcopy(template), deepcopy(x)) for x in sections] for index, section in enumerate(merged, start=1): section["section_index"] = index return merged diff --git a/tests/test_github_templates.py b/tests/test_github_templates.py new file mode 100644 index 00000000..9cd28d28 --- /dev/null +++ b/tests/test_github_templates.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from io import BytesIO +from pathlib import Path +from typing import Any + +import pytest + +from tablassert import ingests +from tablassert.errors import TablassertError + + +def test_parse_github_ref_requires_pinned_ref() -> None: + assert ingests.parse_github_ref("github:org/repo/templates/gene.yaml@v1.0.0") == ("org", "repo", "templates/gene.yaml", "v1.0.0") + for bad in ( + "https://example.org/t.yaml", + "github:org/repo/t.yaml", + "github:org/repo@v1", + "github:org/repo/../../etc/passwd@v1", + "github:org/repo/templates/a.yaml?raw=1@v1", + "github:org/repo/templates/a.yaml@v1#frag", + "github:../repo/t.yaml@v1", + ): + with pytest.raises(TablassertError, match="template-bad-github-ref"): + ingests.parse_github_ref(bad) + + +def test_fetch_github_template_does_not_cache_invalid_yaml_or_http_errors(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from urllib.error import URLError + + cache_dir = tmp_path / "cache" + + def _fail_urlopen(_req: Any, timeout: int = 30) -> Any: + raise URLError("offline") + + monkeypatch.setattr(ingests, "urlopen", _fail_urlopen) + with pytest.raises(TablassertError, match="template-bad-github-ref"): + ingests.fetch_github_template("github:org/repo/t.yaml@v1", cache_dir=cache_dir) + assert not cache_dir.exists() + + class _BadResp(BytesIO): + def __enter__(self) -> _BadResp: + return self + + def __exit__(self, *_args: object) -> None: + return None + + monkeypatch.setattr(ingests, "urlopen", lambda _req, timeout=30: _BadResp(b"- just a list\n")) + with pytest.raises(TablassertError, match="template-bad-github-ref"): + ingests.fetch_github_template("github:org/repo/t.yaml@v1", cache_dir=cache_dir) + assert not cache_dir.exists() + + +def test_to_sections_inherits_and_caches_github_template(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + remote_yaml = b"""template: + source: + kind: text + local: data.tsv + delimiter: "\\t" + url: + - https://example.org/data.tsv + statement: + subject: + method: column + encoding: A + predicate: associated_with + object: + method: column + encoding: B + provenance: + repo: PMID + publication: "12345678" +""" + fetches: list[str] = [] + + class _Resp(BytesIO): + def __enter__(self) -> _Resp: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def _fake_urlopen(req: Any, timeout: int = 30) -> _Resp: + fetches.append(req.full_url) + return _Resp(remote_yaml) + + monkeypatch.setattr(ingests, "urlopen", _fake_urlopen) + cache_dir = tmp_path / "cache" + table_path = tmp_path / "table.yaml" + + instructions = { + "template": {"from": "github:SkyeAv/Tablassert-templates/gene_disease.yaml@abc123", "statement": {"predicate": "related_to"}}, + "sections": [{}], + } + + first = ingests.to_sections(instructions, table_path, cache_dir=cache_dir) + assert len(fetches) == 1 + assert fetches[0] == "https://raw.githubusercontent.com/SkyeAv/Tablassert-templates/abc123/gene_disease.yaml" + assert first[0]["statement"]["predicate"] == "related_to" + assert first[0]["statement"]["subject"]["encoding"] == "A" + + # Second call must hit the local cache without calling urlopen again. + second = ingests.to_sections(instructions, table_path, cache_dir=cache_dir) + assert len(fetches) == 1 + assert second == first