From fb5b9957a641187cd7e904d40b786a54d7f0780f Mon Sep 17 00:00:00 2001 From: Arda Tekin <2003arda2003@gmail.com> Date: Fri, 20 Nov 2026 00:43:53 +0200 Subject: [PATCH] feat: support multiple sources per section with join and antijoin composition --- CHANGELOG.md | 1 + src/tablassert/cli.py | 52 ++++++++----- src/tablassert/errors.py | 6 ++ src/tablassert/lib.py | 121 +++++++++++++++++++++++------ src/tablassert/models.py | 79 +++++++++++++++++-- src/tablassert/runcache.py | 1 + tests/test_cover_models.py | 1 + tests/test_docs_source_of_truth.py | 1 + tests/test_models.py | 1 + tests/test_multi_source_compose.py | 112 ++++++++++++++++++++++++++ 10 files changed, 324 insertions(+), 51 deletions(-) create mode 100644 tests/test_multi_source_compose.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b13f2edd..db726c90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project are documented in this file. ### Added +- **Sections now support multiple named `sources` with lazy `compose` joins and antijoins.** A section can declare `sources: {: , ...}` and `compose: [{left, right, how, on}, ...]` (`inner`, `left`, `right`, `full`, `semi`, `anti`) with namespaced `alias:COL` column addressing (keeping single-`source` configs unchanged and hashing all composed source files in the run cache). ([#271](https://github.com/SkyeAv/Tablassert/issues/271)) - **NDJSON (JSONL) is a fifth table source kind.** `kind: ndjson` scans newline-delimited JSON lazily with `pl.scan_ndjson` and renames its columns positionally to `column_1..` in first-seen key order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Ragged records scan missing keys as nulls instead of dropping rows; an empty file fails fast with a coded `ndjson-empty` error instead of polars' opaque `Cannot infer NDJSON types` compute error. ([#283](https://github.com/SkyeAv/Tablassert/pull/283), [#262](https://github.com/SkyeAv/Tablassert/issues/262)) - **Every pull request must carry a new changelog entry.** A stdlib-only `scripts/check_changelog.py` requires at least one new bullet under the `## Unreleased` heading of `CHANGELOG.md` relative to the pull request's merge base, failing with remediation guidance when whitespace, subheadings, or released-history edits are the only additions; a CI job runs it on every pull request including forks, release-bump and dependency-bump pull requests are exempt by their `chore(release):` / `chore(deps):` title or commit subject, documentation-only pull requests (every changed path is markdown or under `docs/`) are exempt via `--exempt-docs-only`, and a pre-push hook mirrors the same check locally. ([#284](https://github.com/SkyeAv/Tablassert/pull/284), [#282](https://github.com/SkyeAv/Tablassert/issues/282)) - **Arrow IPC (Feather v2) is a fourth table source kind.** `kind: ipc` scans a `.arrow` file lazily with `pl.scan_ipc` and renames its columns positionally to `column_1..` in schema order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Only IPC file layout is supported; a stream-layout file fails fast with a coded `ipc-stream-unsupported` error naming the conversion fix instead of polars' opaque `InvalidFooter` compute error. ([#281](https://github.com/SkyeAv/Tablassert/pull/281), [#264](https://github.com/SkyeAv/Tablassert/issues/264)) diff --git a/src/tablassert/cli.py b/src/tablassert/cli.py index 2fc798b2..99cfd69b 100644 --- a/src/tablassert/cli.py +++ b/src/tablassert/cli.py @@ -115,29 +115,34 @@ def _section_store_key_for_build( from tablassert.utils import file_content_hash, mkhash, section_store_key config_hash: str = mkhash(section) - source: object = section.get("source") - local: object = source.get("local") if isinstance(source, dict) else None - if local is None: + source_entries: list[dict[str, Any]] = [] + if isinstance(section.get("source"), dict): + source_entries.append(section["source"]) + elif isinstance(section.get("sources"), dict): + source_entries.extend(v for v in section["sources"].values() if isinstance(v, dict)) + + locals_list: list[str] = [str(s["local"]) for s in source_entries if s.get("local") is not None] + if not locals_list: return section_store_key(section), config_hash - resolved: Path = Path(str(local)).resolve() config_value: object = section.get("config", "section") section_label: str = f"{Path(str(config_value)).stem} ยท {config_hash[:8]}" - digest: str | None = None - try: - stat = resolved.stat() - except OSError: - # Reuse the contextual helper so missing/unreadable paths never leak a raw OS error. - digest = file_content_hash(resolved, config=configuration_file, section_label=section_label) - return section_store_key(section, content_digest=digest), config_hash - - signature: tuple[Path, int, int] = (resolved, stat.st_mtime_ns, stat.st_size) - if digest is None: - digest = content_hashes.get(signature) - if digest is None: - digest = file_content_hash(resolved, config=configuration_file, section_label=section_label) - content_hashes[signature] = digest - return section_store_key(section, content_digest=digest), config_hash + digests: list[str] = [] + for local in locals_list: + resolved: Path = Path(local).resolve() + try: + stat = resolved.stat() + except OSError: + digests.append(file_content_hash(resolved, config=configuration_file, section_label=section_label)) + continue + signature: tuple[Path, int, int] = (resolved, stat.st_mtime_ns, stat.st_size) + digest: str | None = content_hashes.get(signature) + if digest is None: + digest = file_content_hash(resolved, config=configuration_file, section_label=section_label) + content_hashes[signature] = digest + digests.append(digest) + combined_digest: str = digests[0] if len(digests) == 1 else mkhash(digests) + return section_store_key(section, content_digest=combined_digest), config_hash def _load_table_indexed(args: tuple[int, Path]) -> tuple[int, object]: @@ -363,8 +368,13 @@ def build_graph_pipeline( # each entry records the section's local file name and its validated source URLs. section_sources: list[dict[str, Any]] = [] for s in sections: - src: dict[str, Any] = s.get("source") or {} - section_sources.append({"local": str(src.get("local") or ""), "urls": [str(u) for u in src.get("url") or []]}) + if isinstance(s.get("sources"), dict): + for src in s["sources"].values(): + if isinstance(src, dict): + section_sources.append({"local": str(src.get("local") or ""), "urls": [str(u) for u in src.get("url") or []]}) + else: + src_single: dict[str, Any] = s.get("source") or {} + section_sources.append({"local": str(src_single.get("local") or ""), "urls": [str(u) for u in src_single.get("url") or []]}) # Stage 3/6: build Tcode. progress.stage("Building TCode") diff --git a/src/tablassert/errors.py b/src/tablassert/errors.py index 978659dd..2c46ae18 100644 --- a/src/tablassert/errors.py +++ b/src/tablassert/errors.py @@ -87,6 +87,12 @@ def redact_secrets(text: str, secrets: Sequence[str] = ()) -> str: "qualifier-nullable-literal", "qualifier-duplicated", "qualifier-vocabulary-violation", + "section-source-conflict", + "section-source-required", + "compose-requires-sources", + "compose-unknown-source", + "compose-bad-on", + "encoding-unknown-source", "rig-bad-infores", "rig-bad-artifact-url", "rig-bad-access-location", diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index 0910a27a..9b8e56f0 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -1382,24 +1382,89 @@ def drop_low_number_of_cases(lf: pl.LazyFrame, col: str = "number_of_cases", thr return lf.filter(pl.col(col).cast(pl.Float64, strict=False).ge(threshold).fill_null(True)) -def idxname(col: Any) -> str: - """Convert Excel-style column letters (e.g. ``"AA"``) to a polars-style ``column_`` name. +def _scan_source_dict(src: dict[str, Any]) -> pl.LazyFrame: + """Scan a single source dict (used by ``compose_sources``).""" + kind: str = str(src["kind"]) + local: Path = Path(src["local"]) + if kind == Files.TEXT.value: + lf = csv(local, src.get("delimiter", ",")) + elif kind == Files.EXCEL.value: + lf = excel(local, src.get("sheet", "Sheet1")) + elif kind == Files.PARQUET.value: + lf = parquet(local) + elif kind == Files.IPC.value: + lf = ipc(local) + elif kind == Files.NDJSON.value: + lf = ndjson(local) + else: + raise TablassertError(f"Unsupported source kind: {kind!r}", code="source-file-unreadable") + lf = idx(lf) + if src.get("row_slice"): + lf = crop(lf, src["row_slice"]) + if src.get("rows"): + lf = pick(lf, src["rows"]) + return lf - Args: - col: Excel column letter (or any value whose ``str`` form is one). - Returns: - ``f"column_{idx}"`` where ``idx`` is the 1-based column number. +def compose_sources(sources: dict[str, dict[str, Any]], compose: list[dict[str, Any]] | None) -> pl.LazyFrame: + """Scan named sources and execute lazy polars join/antijoin composition. + + Each source's ``column_`` columns are namespaced to ``__column_`` so + columns never collide across joined tables. The first declared source's ``row_index`` + is preserved as the composed frame's row identity. """ + frames: dict[str, pl.LazyFrame] = {} + for alias, src in sources.items(): + lf = _scan_source_dict(src) + schema_names: list[str] = lf.collect_schema().names() + renames: dict[str, str] = {c: f"{alias}__{c}" for c in schema_names if c.startswith("column_")} + if renames: + lf = lf.rename(renames) + frames[alias] = lf + + primary_alias: str = next(iter(sources)) + composed: pl.LazyFrame = frames[primary_alias] + if not compose: + return composed + + for i, step in enumerate(compose): + left_name: str = str(step["left"]) + right_name: str = str(step["right"]) + how: Literal["inner", "left", "right", "full", "semi", "anti"] = step["how"] + on: str | dict[str, str] = step["on"] + if isinstance(on, str): + left_letter = right_letter = on + else: + left_letter = on["left"] + right_letter = on["right"] + + left_prefix_alias: str = primary_alias if (i > 0 and left_name == "composed") else left_name + left_lf: pl.LazyFrame = composed if (i > 0 or left_name == primary_alias) else frames[left_name] + right_lf: pl.LazyFrame = ( + frames[right_name].drop("extracted_from_row_number") + if "extracted_from_row_number" in frames[right_name].collect_schema().names() + else frames[right_name] + ) + left_on: str = f"{left_prefix_alias}__{idxname(left_letter)}" + right_on: str = f"{right_name}__{idxname(right_letter)}" + composed = left_lf.join(right_lf, left_on=left_on, right_on=right_on, how=how) + return composed + + +def idxname(col: Any, primary_prefix: str = "") -> str: + """Convert Excel-style column letters (e.g. ``"AA"`` or ``"lookup:B"``) to a polars column name.""" scol: str = str(col) + if ":" in scol: + alias, letter = scol.split(":", 1) + return f"{alias}__{idxname(letter)}" idx: int = 0 for char in scol: idx = idx * 26 + (ord(char) - 65 + 1) - return f"column_{idx}" + return f"{primary_prefix}column_{idx}" -def trim(lf: pl.LazyFrame, regex: str = r"^column_\d+$") -> pl.LazyFrame: +def trim(lf: pl.LazyFrame, regex: str = r"^(?:[a-zA-Z_][a-zA-Z0-9_]*__)?column_\d+$") -> pl.LazyFrame: """Remove columns whose names match the Excel-style ``column_`` convention. Args: @@ -1517,9 +1582,10 @@ def encoding(self: Self, x: Encoding, col: str, table_literal: bool = False) -> Flat list of ``Any`` items (tuples or lists of tuples, possibly containing ``None`` placeholders) ready for ``clean`` to filter. """ + primary_prefix: str = f"{next(iter(self.sources))}__" if self.sources else "" return [ (value, (col, x.encoding)) if x.method == EncodingMethods.VALUE else None, - (column, (col, idxname(x.encoding))) if x.method == EncodingMethods.COLUMN else None, + (column, (col, idxname(x.encoding, primary_prefix))) if x.method == EncodingMethods.COLUMN else None, (column, (f"original_{col}", col)) if table_literal else None, (fill, (col, x.fill)) if x.fill else None, (explode, (col, x.explode_by)) if x.explode_by else None, @@ -1561,22 +1627,26 @@ def clean(self: Self, tcode: list[tuple[Callable, Any]]) -> list[tuple[Callable, return [op for x in tcode if x for op in (self.clean(x) if isinstance(x, list) else [x])] def _source_ops(self: Self) -> list[Any]: - """Collect the load/filter/clean/significance ops that precede entity resolution. - - Returns: - Raw (possibly nested, possibly ``None``-padded) op list ready for - ``clean``; order matches the pre-resolution pipeline exactly. - """ + """Collect the load/filter/clean/significance ops that precede entity resolution.""" + if self.sources is not None: + dumped_sources: dict[str, dict[str, Any]] = {k: v.model_dump() for k, v in self.sources.items()} + dumped_compose: list[dict[str, Any]] | None = [step.model_dump() for step in self.compose] if self.compose else None + load_and_filter_ops: list[Any] = [(compose_sources, (dumped_sources, dumped_compose))] + else: + assert self.source is not None + load_and_filter_ops = [ + (csv, (self.source.local, self.source.delimiter)) if self.source.kind == Files.TEXT else None, # pyright: ignore + (excel, (self.source.local, self.source.sheet)) if self.source.kind == Files.EXCEL else None, # pyright: ignore + (parquet, (self.source.local,)) if self.source.kind == Files.PARQUET else None, # pyright: ignore + (ipc, (self.source.local,)) if self.source.kind == Files.IPC else None, # pyright: ignore + (ndjson, (self.source.local,)) if self.source.kind == Files.NDJSON else None, # pyright: ignore + (idx, ()), + (crop, (self.source.row_slice,)) if self.source.row_slice else None, + (pick, (self.source.rows,)) if self.source.rows else None, + [reindex_op(x) for x in self.source.reindex] if self.source.reindex else None, + ] return [ - (csv, (self.source.local, self.source.delimiter)) if self.source.kind == Files.TEXT else None, # pyright: ignore - (excel, (self.source.local, self.source.sheet)) if self.source.kind == Files.EXCEL else None, # pyright: ignore - (parquet, (self.source.local,)) if self.source.kind == Files.PARQUET else None, # pyright: ignore - (ipc, (self.source.local,)) if self.source.kind == Files.IPC else None, # pyright: ignore - (ndjson, (self.source.local,)) if self.source.kind == Files.NDJSON else None, # pyright: ignore - (idx, ()), - (crop, (self.source.row_slice,)) if self.source.row_slice else None, - (pick, (self.source.rows,)) if self.source.rows else None, - [reindex_op(x) for x in self.source.reindex] if self.source.reindex else None, + *load_and_filter_ops, # --head preview: randomly sample min(HEAD_ROWS, height) rows before any encoding/resolve. (head, (HEAD_ROWS,)) if self.head else None, [ @@ -1727,7 +1797,7 @@ def _provenance_ops(self: Self) -> list[Any]: # template into override.sources): an upstream_source_record_urls mapping # re-homes URLs exactly as specified (primary gets nothing), any other override # keeps the legacy primary-entry emission, and the flag injects nothing there. - record_urls: list[str] = [str(u) for u in self.source.url] + record_urls: list[str] = [str(u) for u in self.primary_source.url] upstream_ids = override.upstream_resource_ids if override else upstream_resource_ids(self.provenance.repo) primary_urls: list[str] upstream_urls: dict[str, list[str]] | None @@ -1815,6 +1885,7 @@ def collect(self: Self, db: Path) -> list[tuple[Callable, tuple[Any]]] | Path: csv: "load", excel: "load", parquet: "load", + compose_sources: "load", idx: "load", crop: "filter", pick: "filter", diff --git a/src/tablassert/models.py b/src/tablassert/models.py index f0aecc03..9ebc246d 100644 --- a/src/tablassert/models.py +++ b/src/tablassert/models.py @@ -366,9 +366,11 @@ def reject_removed_list_method(cls, data: Any) -> Any: @model_validator(mode="after") def excel_style_columns(self: Self) -> Self: if self.method == EncodingMethods.COLUMN: - x = self.encoding - if not re.search(r"^[A-Z]{1,3}$", str(x)): - raise TablassertValidationError(f"`encoding` must be an Excel-style column name (A-ZZ), got {x!r}.", code="encoding-bad-excel-column") + x = str(self.encoding) + if not re.search(r"^(?:[a-zA-Z_][a-zA-Z0-9_]*:)?[A-Z]{1,3}$", x): + raise TablassertValidationError( + f"`encoding` must be an Excel-style column name (A-ZZ) or `source:COL`, got {x!r}.", code="encoding-bad-excel-column" + ) return self @@ -991,14 +993,81 @@ def warn_when_the_slot_cannot_reach_the_edge(self) -> Self: return self +class ComposeStep(TablaBase): + """One join or antijoin step composing named section sources.""" + + left: str = Field(..., min_length=1, description="Left source alias (or `'composed'` for the running frame).") + right: str = Field(..., min_length=1, description="Right source alias to join against.") + how: Literal["inner", "left", "right", "full", "semi", "anti"] = Field("left", description="Polars join strategy.") + on: str | dict[str, str] = Field( + ..., description='Join key column letter(s): one Excel column (`A`) for both sides, or `{"left": "A", "right": "B"}`.' + ) + + @field_validator("on", mode="after") + @classmethod + def valid_join_columns(cls, value: str | dict[str, str]) -> str | dict[str, str]: + col_re = re.compile(r"^[A-Z]{1,3}$") + if isinstance(value, str): + if not col_re.match(value): + raise TablassertValidationError(f"`compose.on` column must be an Excel column letter (A-ZZ), got {value!r}.", code="compose-bad-on") + return value + if set(value.keys()) != {"left", "right"} or not all(col_re.match(str(v)) for v in value.values()): + raise TablassertValidationError( + f"`compose.on` mapping must have `left` and `right` Excel column letters (A-ZZ), got {value!r}.", code="compose-bad-on" + ) + return value + + class Section(TablaBase): """Pydantic section model and coercion target for a single table configuration.""" - source: Excel | Text | Parquet | IPC | NDJSON = Field(..., description="Input source definition for reading tabular rows.") + source: Excel | Text | Parquet | IPC | NDJSON | None = Field(None, description="Input source definition for reading tabular rows.") + sources: dict[str, Excel | Text | Parquet | IPC | NDJSON] | None = Field( + None, description="Named input sources for multi-source join/antijoin composition." + ) + compose: list[ComposeStep] | None = Field(None, description="Ordered join/antijoin steps composing `sources`.") statement: Statement = Field(..., description="Subject-object statement mapping for this section.") provenance: Provenance = Field(..., description="Provenance metadata applied to all produced edges.") annotations: list[Annotation] | None = Field(None, description="Optional extra encoded columns added to each row.") + @property + def primary_source(self: Self) -> Excel | Text | Parquet | IPC | NDJSON: + """Return the single `source` or the first entry in `sources`.""" + if self.source is not None: + return self.source + assert self.sources + return next(iter(self.sources.values())) + + @model_validator(mode="after") + def validate_sources_and_compose(self: Self) -> Self: + if self.source is not None and self.sources is not None: + raise TablassertValidationError("`source` and `sources` are mutually exclusive; declare one.", code="section-source-conflict") + if self.source is None and not self.sources: + raise TablassertValidationError("Section requires either `source` or a non-empty `sources` mapping.", code="section-source-required") + if self.compose is not None and self.sources is None: + raise TablassertValidationError("`compose` requires `sources` to be declared on the section.", code="compose-requires-sources") + if self.sources is not None: + known: set[str] = set(self.sources.keys()) + for i, step in enumerate(self.compose or ()): + if step.left not in known and not (i > 0 and step.left == "composed"): + raise TablassertValidationError( + f"`compose[{i}].left` references unknown source {step.left!r} (known: {sorted(known)}).", code="compose-unknown-source" + ) + if step.right not in known: + raise TablassertValidationError( + f"`compose[{i}].right` references unknown source {step.right!r} (known: {sorted(known)}).", code="compose-unknown-source" + ) + encodings: list[Encoding] = [self.statement.subject, self.statement.object, *(self.statement.qualifiers or ()), *(self.annotations or ())] + for enc in encodings: + if enc.method == EncodingMethods.COLUMN and ":" in str(enc.encoding): + alias, _ = str(enc.encoding).split(":", 1) + if alias not in known: + raise TablassertValidationError( + f"Column encoding {enc.encoding!r} references unknown source {alias!r} (known: {sorted(known)}).", + code="encoding-unknown-source", + ) + return self + @model_validator(mode="after") def drop_unpaired_effect_annotations(self) -> Self: """Drop an ``effect_size`` / ``effect_type`` annotation declared without its sibling. @@ -1030,7 +1099,7 @@ def drop_unpaired_effect_annotations(self) -> Self: dropped_target: str = "effect_size" if has_size else "effect_type" sibling: str = "effect_type" if has_size else "effect_size" - where: str = _section_source_label(self.source) + where: str = _section_source_label(self.primary_source) kept: list[Annotation] = [] for annotation in annotations: name: str = str(annotation.annotation) diff --git a/src/tablassert/runcache.py b/src/tablassert/runcache.py index 149d4e30..ee1ba024 100644 --- a/src/tablassert/runcache.py +++ b/src/tablassert/runcache.py @@ -255,6 +255,7 @@ def prefix_digest(ops: list[tuple[Callable, tuple[Any, ...]]]) -> str: lib.parquet: 2, lib.ipc: 2, lib.ndjson: 2, + lib.compose_sources: 4, # Filter: single lazy row predicates/slices. lib.idx: 1, lib.crop: 1, diff --git a/tests/test_cover_models.py b/tests/test_cover_models.py index 54bb1e70..a06f6399 100644 --- a/tests/test_cover_models.py +++ b/tests/test_cover_models.py @@ -131,6 +131,7 @@ def test_section_accepts_all_source_kinds() -> None: for kind, model in [("text", Text), ("excel", Excel), ("parquet", Parquet), ("ipc", IPC), ("ndjson", NDJSON)]: payload: dict[str, Any] = {**base, "source": {"kind": kind, "local": f"./table.{kind}", "url": [f"https://example.com/table.{kind}"]}} section: Section = Section.model_validate(payload) + assert section.source is not None assert section.source.kind == kind assert type(section.source) is model diff --git a/tests/test_docs_source_of_truth.py b/tests/test_docs_source_of_truth.py index 792472ee..402c78a3 100644 --- a/tests/test_docs_source_of_truth.py +++ b/tests/test_docs_source_of_truth.py @@ -612,6 +612,7 @@ def location(self) -> str: # `ModelTableBinding` above, or add its name here. UNBOUND_MODELS: frozenset[str] = frozenset( { + "ComposeStep", "TablaBase", "BaseSource", "Encoding", diff --git a/tests/test_models.py b/tests/test_models.py index 0a4d9482..75a88425 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -38,6 +38,7 @@ def test_section_from_minimal_yaml(fixtures_path: Path) -> None: """valid minimal text section.""" data: Any = from_yaml(fixtures_path / "minimal_section.yaml") section: Section = Section(**data) # pyright: ignore + assert section.source is not None assert section.source.kind == "text" assert section.statement.subject.encoding == "BRCA1" assert section.statement.object.encoding == "TP53" diff --git a/tests/test_multi_source_compose.py b/tests/test_multi_source_compose.py new file mode 100644 index 00000000..5c766a3a --- /dev/null +++ b/tests/test_multi_source_compose.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from pathlib import Path + +import pydantic +import pytest + +from tablassert.lib import Tcode, compose_sources, idxname, trim +from tablassert.models import Section + + +def test_section_source_and_compose_validation() -> None: + base_stmt = { + "subject": {"method": "column", "encoding": "A"}, + "predicate": "associated_with", + "object": {"method": "column", "encoding": "lookup:B"}, + } + base_prov = {"repo": "PMID", "publication": "12345678"} + + with pytest.raises(pydantic.ValidationError, match="section-source-conflict"): + Section.model_validate( + { + "source": {"kind": "text", "local": "a.tsv", "url": ["https://example.org/a.tsv"]}, + "sources": {"facts": {"kind": "text", "local": "a.tsv", "url": ["https://example.org/a.tsv"]}}, + "statement": base_stmt, + "provenance": base_prov, + } + ) + + with pytest.raises(pydantic.ValidationError, match="section-source-required"): + Section.model_validate({"statement": base_stmt, "provenance": base_prov}) + + with pytest.raises(pydantic.ValidationError, match="compose-unknown-source"): + Section.model_validate( + { + "sources": {"facts": {"kind": "text", "local": "a.tsv", "url": ["https://example.org/a.tsv"]}}, + "compose": [{"left": "facts", "right": "missing", "how": "left", "on": "A"}], + "statement": { + "subject": {"method": "column", "encoding": "A"}, + "predicate": "associated_with", + "object": {"method": "column", "encoding": "B"}, + }, + "provenance": base_prov, + } + ) + + with pytest.raises(pydantic.ValidationError, match="encoding-unknown-source"): + Section.model_validate( + { + "sources": {"facts": {"kind": "text", "local": "a.tsv", "url": ["https://example.org/a.tsv"]}}, + "statement": base_stmt, + "provenance": base_prov, + } + ) + + +def test_compose_sources_left_join_and_antijoin(tmp_path: Path) -> None: + facts = tmp_path / "facts.tsv" + facts.write_text("G1\tD1\nG2\tD2\nG3\tD3\n", encoding="utf-8") + lookup = tmp_path / "lookup.tsv" + lookup.write_text("G1\tBRCA1\nG2\tTP53\n", encoding="utf-8") + blocklist = tmp_path / "block.tsv" + blocklist.write_text("G2\n", encoding="utf-8") + + sources = { + "facts": {"kind": "text", "local": str(facts), "delimiter": "\t", "url": ["https://example.org/facts.tsv"]}, + "lookup": {"kind": "text", "local": str(lookup), "delimiter": "\t", "url": ["https://example.org/lookup.tsv"]}, + "block": {"kind": "text", "local": str(blocklist), "delimiter": "\t", "url": ["https://example.org/block.tsv"]}, + } + compose = [{"left": "facts", "right": "lookup", "how": "left", "on": "A"}, {"left": "composed", "right": "block", "how": "anti", "on": "A"}] + + lf = compose_sources(sources, compose) + df = lf.collect() + assert df["facts__column_1"].to_list() == ["G1", "G3"] + assert df["lookup__column_2"].to_list() == ["BRCA1", None] + assert df["extracted_from_row_number"].to_list() == [1, 3] + + trimmed = trim(lf).collect() + assert trimmed.columns == ["extracted_from_row_number"] + + +def test_idxname_supports_namespaced_and_bare_columns() -> None: + assert idxname("A") == "column_1" + assert idxname("lookup:B") == "lookup__column_2" + assert idxname("A", primary_prefix="facts__") == "facts__column_1" + + +def test_tcode_multi_source_builds_source_ops(tmp_path: Path) -> None: + facts = tmp_path / "facts.tsv" + facts.write_text("G1\tD1\n", encoding="utf-8") + lookup = tmp_path / "lookup.tsv" + lookup.write_text("G1\tBRCA1\n", encoding="utf-8") + + tc = Tcode.model_validate( + { + "config": tmp_path / "t.yaml", + "store": tmp_path / "s.parquet", + "sources": { + "facts": {"kind": "text", "local": facts, "delimiter": "\t", "url": ["https://example.org/facts.tsv"]}, + "lookup": {"kind": "text", "local": lookup, "delimiter": "\t", "url": ["https://example.org/lookup.tsv"]}, + }, + "compose": [{"left": "facts", "right": "lookup", "how": "inner", "on": "A"}], + "statement": { + "subject": {"method": "column", "encoding": "lookup:B"}, + "predicate": "associated_with", + "object": {"method": "column", "encoding": "B"}, + }, + "provenance": {"repo": "PMID", "publication": "12345678"}, + } + ) + ops = tc._source_ops() + assert ops[0][0] is compose_sources