Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ All notable changes to this project are documented in this file.

### Added

- **Table configs can inherit `template:` definitions from pinned GitHub YAML files.** Setting `template.from: github:<owner>/<repo>/<path>@<ref>` (with optional `select:`) fetches and caches the remote YAML under `.tablassert/templates/`, parses it with `CSafeLoader`, and deep-merges it under the local template via `fastmerge` so offline cached builds match inlined equivalents. ([#277](https://github.com/SkyeAv/Tablassert/issues/277))
- **NDJSON (JSONL) is a fifth table source kind.** `kind: ndjson` scans newline-delimited JSON lazily with `pl.scan_ndjson` and renames its columns positionally to `column_1..` in first-seen key order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Ragged records scan missing keys as nulls instead of dropping rows; an empty file fails fast with a coded `ndjson-empty` error instead of polars' opaque `Cannot infer NDJSON types` compute error. ([#283](https://github.com/SkyeAv/Tablassert/pull/283), [#262](https://github.com/SkyeAv/Tablassert/issues/262))
- **Every pull request must carry a new changelog entry.** A stdlib-only `scripts/check_changelog.py` requires at least one new bullet under the `## Unreleased` heading of `CHANGELOG.md` relative to the pull request's merge base, failing with remediation guidance when whitespace, subheadings, or released-history edits are the only additions; a CI job runs it on every pull request including forks, release-bump and dependency-bump pull requests are exempt by their `chore(release):` / `chore(deps):` title or commit subject, documentation-only pull requests (every changed path is markdown or under `docs/`) are exempt via `--exempt-docs-only`, and a pre-push hook mirrors the same check locally. ([#284](https://github.com/SkyeAv/Tablassert/pull/284), [#282](https://github.com/SkyeAv/Tablassert/issues/282))
- **Arrow IPC (Feather v2) is a fourth table source kind.** `kind: ipc` scans a `.arrow` file lazily with `pl.scan_ipc` and renames its columns positionally to `column_1..` in schema order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Only IPC file layout is supported; a stream-layout file fails fast with a coded `ipc-stream-unsupported` error naming the conversion fix instead of polars' opaque `InvalidFooter` compute error. ([#281](https://github.com/SkyeAv/Tablassert/pull/281), [#264](https://github.com/SkyeAv/Tablassert/issues/264))
Expand Down
1 change: 1 addition & 0 deletions src/tablassert/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ def redact_secrets(text: str, secrets: Sequence[str] = ()) -> str:
"rig-bad-infores",
"rig-bad-artifact-url",
"rig-bad-access-location",
"template-bad-github-ref",
"rig-terms-empty",
"rig-legacy-keys",
"rig-validation-failed",
Expand Down
93 changes: 78 additions & 15 deletions src/tablassert/ingests.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
from __future__ import annotations

import hashlib
import re
from copy import deepcopy
from pathlib import Path
from typing import Any
from urllib.error import URLError
from urllib.request import Request, urlopen

import yaml
from yaml import CSafeLoader

from tablassert.errors import TablassertError


def fastmerge(a: list[Any] | dict[str, Any], b: list[Any] | dict[str, Any]) -> Any:
"""Recursively merge ``b`` into ``a`` in place and return the result.
Expand Down Expand Up @@ -82,27 +88,84 @@ def to_yaml(p: Path, data: object) -> None:
yaml.dump(data, f, Dumper=_IndentedSafeDumper, sort_keys=False, default_flow_style=False, allow_unicode=True, width=120)


def to_sections(instructions: dict[str, Any], table: Path) -> list[dict[str, Any]]:
_GITHUB_SLUG_RE: re.Pattern[str] = re.compile(r"^[A-Za-z0-9_.-]+$")


def parse_github_ref(spec: str) -> tuple[str, str, str, str]:
"""Parse and validate a pinned ``github:<owner>/<repo>/<path>@<ref>`` template reference."""
if not spec.startswith("github:") or "@" not in spec:
raise TablassertError(f"Remote template `from` must use `github:<owner>/<repo>/<path>@<ref>`, got {spec!r}.", code="template-bad-github-ref")
body, ref = spec.removeprefix("github:").rsplit("@", 1)
parts: list[str] = body.split("/", 2)
if len(parts) != 3 or not all(parts) or not ref:
raise TablassertError(f"Remote template `from` must use `github:<owner>/<repo>/<path>@<ref>`, got {spec!r}.", code="template-bad-github-ref")
owner, repo, path = parts
if not _GITHUB_SLUG_RE.match(owner) or owner in (".", "..") or not _GITHUB_SLUG_RE.match(repo) or repo in (".", ".."):
raise TablassertError(f"Invalid GitHub owner/repo in template ref {spec!r}.", code="template-bad-github-ref")
path_segments: list[str] = path.split("/")
if any(seg in ("", ".", "..") for seg in path_segments) or any(c in path for c in ("?", "#", "\\")) or any(c.isspace() for c in path):
raise TablassertError(f"Invalid file path in GitHub template ref {spec!r}.", code="template-bad-github-ref")
if ".." in ref or ref.startswith("/") or ref.endswith("/") or any(c in ref for c in ("?", "#", "\\")) or any(c.isspace() for c in ref):
raise TablassertError(f"Invalid git ref in GitHub template ref {spec!r}.", code="template-bad-github-ref")
return owner, repo, path, ref


def fetch_github_template(spec: str, *, cache_dir: Path | None = None) -> tuple[dict[str, Any], str]:
"""Fetch (or read from local cache) a GitHub-hosted YAML template using CSafeLoader."""
owner, repo, path, ref = parse_github_ref(spec)
cache_root: Path = cache_dir or Path(".tablassert/templates")
key: str = hashlib.sha256(f"{owner}/{repo}/{ref}/{path}".encode()).hexdigest()
cached_file: Path = cache_root / f"{key}.yaml"

from_cache: bool = cached_file.is_file()
if from_cache:
raw_bytes: bytes = cached_file.read_bytes()
else:
url: str = f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}"
req = Request(url, headers={"User-Agent": "tablassert"})
try:
with urlopen(req, timeout=30) as resp:
raw_bytes = resp.read()
except (URLError, OSError) as exc:
raise TablassertError(f"Failed to fetch remote template {spec!r}: {exc}", code="template-bad-github-ref") from exc

try:
parsed: object = yaml.load(raw_bytes.decode("utf-8"), Loader=CSafeLoader)
except (UnicodeDecodeError, yaml.YAMLError) as exc:
raise TablassertError(f"Remote template {spec!r} failed YAML parsing: {exc}", code="template-bad-github-ref") from exc
if not isinstance(parsed, dict):
raise TablassertError(f"Remote template {spec!r} did not parse as a YAML mapping.", code="template-bad-github-ref")

if not from_cache:
cache_root.mkdir(parents=True, exist_ok=True)
cached_file.write_bytes(raw_bytes)
digest: str = hashlib.sha256(raw_bytes).hexdigest()
return parsed, digest


def to_sections(instructions: dict[str, Any], table: Path, *, cache_dir: Path | None = None) -> list[dict[str, Any]]:
"""Expand a table config dict into a list of section dicts.

Each section is the deep merge of the table-level ``template`` over a
per-section entry. The originating ``table`` path is stamped into each
section as ``config``, and the section's 1-based position in the file
(document order) as ``section_index`` so provenance can name the inlined
Study after its origin ("<table config> SECTION <n>").

Args:
instructions: Parsed YAML config with optional ``template`` and
``sections`` keys.
table: Path of the source table file (recorded on each section).

Returns:
One merged dict per section.
per-section entry. If ``template.from`` references a pinned GitHub YAML
(``github:owner/repo/path.yaml@ref``), it is fetched/cached and merged under
the local ``template``.
"""
template: dict[str, Any] = instructions.get("template", {})
template: dict[str, Any] = deepcopy(instructions.get("template", {}))
remote_ref: object = template.pop("from", None)
select_key: str = str(template.pop("select", "template"))
if remote_ref is not None:
if not isinstance(remote_ref, str):
raise TablassertError(f"Remote template `from` must be a string, got {type(remote_ref).__name__}.", code="template-bad-github-ref")
remote_doc, _remote_digest = fetch_github_template(remote_ref, cache_dir=cache_dir)
selected: object = remote_doc.get(select_key)
if not isinstance(selected, dict):
raise TablassertError(f"Remote template {remote_ref!r} has no mapping under key {select_key!r}.", code="template-bad-github-ref")
template = fastmerge(deepcopy(selected), template)

template["config"] = table
sections: list[dict[str, Any]] = instructions.get("sections", [{}])
merged: list[dict[str, Any]] = [fastmerge(deepcopy(template), x) for x in sections]
merged: list[dict[str, Any]] = [fastmerge(deepcopy(template), deepcopy(x)) for x in sections]
for index, section in enumerate(merged, start=1):
section["section_index"] = index
return merged
105 changes: 105 additions & 0 deletions tests/test_github_templates.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
from __future__ import annotations

from io import BytesIO
from pathlib import Path
from typing import Any

import pytest

from tablassert import ingests
from tablassert.errors import TablassertError


def test_parse_github_ref_requires_pinned_ref() -> None:
assert ingests.parse_github_ref("github:org/repo/templates/gene.yaml@v1.0.0") == ("org", "repo", "templates/gene.yaml", "v1.0.0")
for bad in (
"https://example.org/t.yaml",
"github:org/repo/t.yaml",
"github:org/repo@v1",
"github:org/repo/../../etc/passwd@v1",
"github:org/repo/templates/a.yaml?raw=1@v1",
"github:org/repo/templates/a.yaml@v1#frag",
"github:../repo/t.yaml@v1",
):
with pytest.raises(TablassertError, match="template-bad-github-ref"):
ingests.parse_github_ref(bad)


def test_fetch_github_template_does_not_cache_invalid_yaml_or_http_errors(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
from urllib.error import URLError

cache_dir = tmp_path / "cache"

def _fail_urlopen(_req: Any, timeout: int = 30) -> Any:
raise URLError("offline")

monkeypatch.setattr(ingests, "urlopen", _fail_urlopen)
with pytest.raises(TablassertError, match="template-bad-github-ref"):
ingests.fetch_github_template("github:org/repo/t.yaml@v1", cache_dir=cache_dir)
assert not cache_dir.exists()

class _BadResp(BytesIO):
def __enter__(self) -> _BadResp:
return self

def __exit__(self, *_args: object) -> None:
return None

monkeypatch.setattr(ingests, "urlopen", lambda _req, timeout=30: _BadResp(b"- just a list\n"))
with pytest.raises(TablassertError, match="template-bad-github-ref"):
ingests.fetch_github_template("github:org/repo/t.yaml@v1", cache_dir=cache_dir)
assert not cache_dir.exists()


def test_to_sections_inherits_and_caches_github_template(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
remote_yaml = b"""template:
source:
kind: text
local: data.tsv
delimiter: "\\t"
url:
- https://example.org/data.tsv
statement:
subject:
method: column
encoding: A
predicate: associated_with
object:
method: column
encoding: B
provenance:
repo: PMID
publication: "12345678"
"""
fetches: list[str] = []

class _Resp(BytesIO):
def __enter__(self) -> _Resp:
return self

def __exit__(self, *_args: object) -> None:
return None

def _fake_urlopen(req: Any, timeout: int = 30) -> _Resp:
fetches.append(req.full_url)
return _Resp(remote_yaml)

monkeypatch.setattr(ingests, "urlopen", _fake_urlopen)
cache_dir = tmp_path / "cache"
table_path = tmp_path / "table.yaml"

instructions = {
"template": {"from": "github:SkyeAv/Tablassert-templates/gene_disease.yaml@abc123", "statement": {"predicate": "related_to"}},
"sections": [{}],
}

first = ingests.to_sections(instructions, table_path, cache_dir=cache_dir)
assert len(fetches) == 1
assert fetches[0] == "https://raw.githubusercontent.com/SkyeAv/Tablassert-templates/abc123/gene_disease.yaml"
assert first[0]["statement"]["predicate"] == "related_to"
assert first[0]["statement"]["subject"]["encoding"] == "A"

# Second call must hit the local cache without calling urlopen again.
second = ingests.to_sections(instructions, table_path, cache_dir=cache_dir)
assert len(fetches) == 1
assert second == first