Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ All notable changes to this project are documented in this file.

### Added

- **Sections now support multiple named `sources` with lazy `compose` joins and antijoins.** A section can declare `sources: {<alias>: <Source>, ...}` and `compose: [{left, right, how, on}, ...]` (`inner`, `left`, `right`, `full`, `semi`, `anti`) with namespaced `alias:COL` column addressing (keeping single-`source` configs unchanged and hashing all composed source files in the run cache). ([#271](https://github.com/SkyeAv/Tablassert/issues/271))
- **NDJSON (JSONL) is a fifth table source kind.** `kind: ndjson` scans newline-delimited JSON lazily with `pl.scan_ndjson` and renames its columns positionally to `column_1..` in first-seen key order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Ragged records scan missing keys as nulls instead of dropping rows; an empty file fails fast with a coded `ndjson-empty` error instead of polars' opaque `Cannot infer NDJSON types` compute error. ([#283](https://github.com/SkyeAv/Tablassert/pull/283), [#262](https://github.com/SkyeAv/Tablassert/issues/262))
- **Every pull request must carry a new changelog entry.** A stdlib-only `scripts/check_changelog.py` requires at least one new bullet under the `## Unreleased` heading of `CHANGELOG.md` relative to the pull request's merge base, failing with remediation guidance when whitespace, subheadings, or released-history edits are the only additions; a CI job runs it on every pull request including forks, release-bump and dependency-bump pull requests are exempt by their `chore(release):` / `chore(deps):` title or commit subject, documentation-only pull requests (every changed path is markdown or under `docs/`) are exempt via `--exempt-docs-only`, and a pre-push hook mirrors the same check locally. ([#284](https://github.com/SkyeAv/Tablassert/pull/284), [#282](https://github.com/SkyeAv/Tablassert/issues/282))
- **Arrow IPC (Feather v2) is a fourth table source kind.** `kind: ipc` scans a `.arrow` file lazily with `pl.scan_ipc` and renames its columns positionally to `column_1..` in schema order, matching the `parquet` reader contract so `method: column` encodings keep using Excel-style letters. Only IPC file layout is supported; a stream-layout file fails fast with a coded `ipc-stream-unsupported` error naming the conversion fix instead of polars' opaque `InvalidFooter` compute error. ([#281](https://github.com/SkyeAv/Tablassert/pull/281), [#264](https://github.com/SkyeAv/Tablassert/issues/264))
Expand Down
52 changes: 31 additions & 21 deletions src/tablassert/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,29 +115,34 @@ def _section_store_key_for_build(
from tablassert.utils import file_content_hash, mkhash, section_store_key

config_hash: str = mkhash(section)
source: object = section.get("source")
local: object = source.get("local") if isinstance(source, dict) else None
if local is None:
source_entries: list[dict[str, Any]] = []
if isinstance(section.get("source"), dict):
source_entries.append(section["source"])
elif isinstance(section.get("sources"), dict):
source_entries.extend(v for v in section["sources"].values() if isinstance(v, dict))

locals_list: list[str] = [str(s["local"]) for s in source_entries if s.get("local") is not None]
if not locals_list:
return section_store_key(section), config_hash

resolved: Path = Path(str(local)).resolve()
config_value: object = section.get("config", "section")
section_label: str = f"{Path(str(config_value)).stem} · {config_hash[:8]}"
digest: str | None = None
try:
stat = resolved.stat()
except OSError:
# Reuse the contextual helper so missing/unreadable paths never leak a raw OS error.
digest = file_content_hash(resolved, config=configuration_file, section_label=section_label)
return section_store_key(section, content_digest=digest), config_hash

signature: tuple[Path, int, int] = (resolved, stat.st_mtime_ns, stat.st_size)
if digest is None:
digest = content_hashes.get(signature)
if digest is None:
digest = file_content_hash(resolved, config=configuration_file, section_label=section_label)
content_hashes[signature] = digest
return section_store_key(section, content_digest=digest), config_hash
digests: list[str] = []
for local in locals_list:
resolved: Path = Path(local).resolve()
try:
stat = resolved.stat()
except OSError:
digests.append(file_content_hash(resolved, config=configuration_file, section_label=section_label))
continue
signature: tuple[Path, int, int] = (resolved, stat.st_mtime_ns, stat.st_size)
digest: str | None = content_hashes.get(signature)
if digest is None:
digest = file_content_hash(resolved, config=configuration_file, section_label=section_label)
content_hashes[signature] = digest
digests.append(digest)
combined_digest: str = digests[0] if len(digests) == 1 else mkhash(digests)
return section_store_key(section, content_digest=combined_digest), config_hash


def _load_table_indexed(args: tuple[int, Path]) -> tuple[int, object]:
Expand Down Expand Up @@ -363,8 +368,13 @@ def build_graph_pipeline(
# each entry records the section's local file name and its validated source URLs.
section_sources: list[dict[str, Any]] = []
for s in sections:
src: dict[str, Any] = s.get("source") or {}
section_sources.append({"local": str(src.get("local") or ""), "urls": [str(u) for u in src.get("url") or []]})
if isinstance(s.get("sources"), dict):
for src in s["sources"].values():
if isinstance(src, dict):
section_sources.append({"local": str(src.get("local") or ""), "urls": [str(u) for u in src.get("url") or []]})
else:
src_single: dict[str, Any] = s.get("source") or {}
section_sources.append({"local": str(src_single.get("local") or ""), "urls": [str(u) for u in src_single.get("url") or []]})

# Stage 3/6: build Tcode.
progress.stage("Building TCode")
Expand Down
6 changes: 6 additions & 0 deletions src/tablassert/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,12 @@ def redact_secrets(text: str, secrets: Sequence[str] = ()) -> str:
"qualifier-nullable-literal",
"qualifier-duplicated",
"qualifier-vocabulary-violation",
"section-source-conflict",
"section-source-required",
"compose-requires-sources",
"compose-unknown-source",
"compose-bad-on",
"encoding-unknown-source",
"rig-bad-infores",
"rig-bad-artifact-url",
"rig-bad-access-location",
Expand Down
121 changes: 96 additions & 25 deletions src/tablassert/lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -1382,24 +1382,89 @@ def drop_low_number_of_cases(lf: pl.LazyFrame, col: str = "number_of_cases", thr
return lf.filter(pl.col(col).cast(pl.Float64, strict=False).ge(threshold).fill_null(True))


def idxname(col: Any) -> str:
"""Convert Excel-style column letters (e.g. ``"AA"``) to a polars-style ``column_<n>`` name.
def _scan_source_dict(src: dict[str, Any]) -> pl.LazyFrame:
"""Scan a single source dict (used by ``compose_sources``)."""
kind: str = str(src["kind"])
local: Path = Path(src["local"])
if kind == Files.TEXT.value:
lf = csv(local, src.get("delimiter", ","))
elif kind == Files.EXCEL.value:
lf = excel(local, src.get("sheet", "Sheet1"))
elif kind == Files.PARQUET.value:
lf = parquet(local)
elif kind == Files.IPC.value:
lf = ipc(local)
elif kind == Files.NDJSON.value:
lf = ndjson(local)
else:
raise TablassertError(f"Unsupported source kind: {kind!r}", code="source-file-unreadable")
lf = idx(lf)
if src.get("row_slice"):
lf = crop(lf, src["row_slice"])
if src.get("rows"):
lf = pick(lf, src["rows"])
return lf

Args:
col: Excel column letter (or any value whose ``str`` form is one).

Returns:
``f"column_{idx}"`` where ``idx`` is the 1-based column number.
def compose_sources(sources: dict[str, dict[str, Any]], compose: list[dict[str, Any]] | None) -> pl.LazyFrame:
"""Scan named sources and execute lazy polars join/antijoin composition.

Each source's ``column_<n>`` columns are namespaced to ``<alias>__column_<n>`` so
columns never collide across joined tables. The first declared source's ``row_index``
is preserved as the composed frame's row identity.
"""
frames: dict[str, pl.LazyFrame] = {}
for alias, src in sources.items():
lf = _scan_source_dict(src)
schema_names: list[str] = lf.collect_schema().names()
renames: dict[str, str] = {c: f"{alias}__{c}" for c in schema_names if c.startswith("column_")}
if renames:
lf = lf.rename(renames)
frames[alias] = lf

primary_alias: str = next(iter(sources))
composed: pl.LazyFrame = frames[primary_alias]
if not compose:
return composed

for i, step in enumerate(compose):
left_name: str = str(step["left"])
right_name: str = str(step["right"])
how: Literal["inner", "left", "right", "full", "semi", "anti"] = step["how"]
on: str | dict[str, str] = step["on"]
if isinstance(on, str):
left_letter = right_letter = on
else:
left_letter = on["left"]
right_letter = on["right"]

left_prefix_alias: str = primary_alias if (i > 0 and left_name == "composed") else left_name
left_lf: pl.LazyFrame = composed if (i > 0 or left_name == primary_alias) else frames[left_name]
right_lf: pl.LazyFrame = (
frames[right_name].drop("extracted_from_row_number")
if "extracted_from_row_number" in frames[right_name].collect_schema().names()
else frames[right_name]
)
left_on: str = f"{left_prefix_alias}__{idxname(left_letter)}"
right_on: str = f"{right_name}__{idxname(right_letter)}"
composed = left_lf.join(right_lf, left_on=left_on, right_on=right_on, how=how)
return composed


def idxname(col: Any, primary_prefix: str = "") -> str:
"""Convert Excel-style column letters (e.g. ``"AA"`` or ``"lookup:B"``) to a polars column name."""
scol: str = str(col)
if ":" in scol:
alias, letter = scol.split(":", 1)
return f"{alias}__{idxname(letter)}"
idx: int = 0
for char in scol:
idx = idx * 26 + (ord(char) - 65 + 1)

return f"column_{idx}"
return f"{primary_prefix}column_{idx}"


def trim(lf: pl.LazyFrame, regex: str = r"^column_\d+$") -> pl.LazyFrame:
def trim(lf: pl.LazyFrame, regex: str = r"^(?:[a-zA-Z_][a-zA-Z0-9_]*__)?column_\d+$") -> pl.LazyFrame:
"""Remove columns whose names match the Excel-style ``column_<n>`` convention.

Args:
Expand Down Expand Up @@ -1517,9 +1582,10 @@ def encoding(self: Self, x: Encoding, col: str, table_literal: bool = False) ->
Flat list of ``Any`` items (tuples or lists of tuples, possibly
containing ``None`` placeholders) ready for ``clean`` to filter.
"""
primary_prefix: str = f"{next(iter(self.sources))}__" if self.sources else ""
return [
(value, (col, x.encoding)) if x.method == EncodingMethods.VALUE else None,
(column, (col, idxname(x.encoding))) if x.method == EncodingMethods.COLUMN else None,
(column, (col, idxname(x.encoding, primary_prefix))) if x.method == EncodingMethods.COLUMN else None,
(column, (f"original_{col}", col)) if table_literal else None,
(fill, (col, x.fill)) if x.fill else None,
(explode, (col, x.explode_by)) if x.explode_by else None,
Expand Down Expand Up @@ -1561,22 +1627,26 @@ def clean(self: Self, tcode: list[tuple[Callable, Any]]) -> list[tuple[Callable,
return [op for x in tcode if x for op in (self.clean(x) if isinstance(x, list) else [x])]

def _source_ops(self: Self) -> list[Any]:
"""Collect the load/filter/clean/significance ops that precede entity resolution.

Returns:
Raw (possibly nested, possibly ``None``-padded) op list ready for
``clean``; order matches the pre-resolution pipeline exactly.
"""
"""Collect the load/filter/clean/significance ops that precede entity resolution."""
if self.sources is not None:
dumped_sources: dict[str, dict[str, Any]] = {k: v.model_dump() for k, v in self.sources.items()}
dumped_compose: list[dict[str, Any]] | None = [step.model_dump() for step in self.compose] if self.compose else None
load_and_filter_ops: list[Any] = [(compose_sources, (dumped_sources, dumped_compose))]
else:
assert self.source is not None
load_and_filter_ops = [
(csv, (self.source.local, self.source.delimiter)) if self.source.kind == Files.TEXT else None, # pyright: ignore
(excel, (self.source.local, self.source.sheet)) if self.source.kind == Files.EXCEL else None, # pyright: ignore
(parquet, (self.source.local,)) if self.source.kind == Files.PARQUET else None, # pyright: ignore
(ipc, (self.source.local,)) if self.source.kind == Files.IPC else None, # pyright: ignore
(ndjson, (self.source.local,)) if self.source.kind == Files.NDJSON else None, # pyright: ignore
(idx, ()),
(crop, (self.source.row_slice,)) if self.source.row_slice else None,
(pick, (self.source.rows,)) if self.source.rows else None,
[reindex_op(x) for x in self.source.reindex] if self.source.reindex else None,
]
return [
(csv, (self.source.local, self.source.delimiter)) if self.source.kind == Files.TEXT else None, # pyright: ignore
(excel, (self.source.local, self.source.sheet)) if self.source.kind == Files.EXCEL else None, # pyright: ignore
(parquet, (self.source.local,)) if self.source.kind == Files.PARQUET else None, # pyright: ignore
(ipc, (self.source.local,)) if self.source.kind == Files.IPC else None, # pyright: ignore
(ndjson, (self.source.local,)) if self.source.kind == Files.NDJSON else None, # pyright: ignore
(idx, ()),
(crop, (self.source.row_slice,)) if self.source.row_slice else None,
(pick, (self.source.rows,)) if self.source.rows else None,
[reindex_op(x) for x in self.source.reindex] if self.source.reindex else None,
*load_and_filter_ops,
# --head preview: randomly sample min(HEAD_ROWS, height) rows before any encoding/resolve.
(head, (HEAD_ROWS,)) if self.head else None,
[
Expand Down Expand Up @@ -1727,7 +1797,7 @@ def _provenance_ops(self: Self) -> list[Any]:
# template into override.sources): an upstream_source_record_urls mapping
# re-homes URLs exactly as specified (primary gets nothing), any other override
# keeps the legacy primary-entry emission, and the flag injects nothing there.
record_urls: list[str] = [str(u) for u in self.source.url]
record_urls: list[str] = [str(u) for u in self.primary_source.url]
upstream_ids = override.upstream_resource_ids if override else upstream_resource_ids(self.provenance.repo)
primary_urls: list[str]
upstream_urls: dict[str, list[str]] | None
Expand Down Expand Up @@ -1815,6 +1885,7 @@ def collect(self: Self, db: Path) -> list[tuple[Callable, tuple[Any]]] | Path:
csv: "load",
excel: "load",
parquet: "load",
compose_sources: "load",
idx: "load",
crop: "filter",
pick: "filter",
Expand Down
Loading