diff --git a/README.md b/README.md index c8ffad76c..3e9e04305 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ combines the two, and the files next to each: - **`patch/*.patch` and `tests/`:** per package, the distribution's file replaces the shared one. In tests, `- if: ros_distro in ["humble", "jazzy"]` blocks select distribution-specific parts. - **`vinca_pinning.yaml`:** the shared pinning, adjusted by `distro.yaml`: - `conda_forge_pinning_version` and `conda_forge_migrations` keep a distribution on its own conda-forge pinning, for example until its next full rebuild; - - `pinning_overrides` replace shared pins by name, for example jazzy's Python 3.12. + - `pinning_overrides` replace shared pins by name, for example jazzy's Python 3.12. Items written as a block list keep their conda-build selectors (`- 3.12.* # [not riscv64]`). - `pixi run rs render-pinning` turns this into the committed `conda_build_config.yaml`. A fix that applies to every distribution goes into `shared/`, a version-specific one into `distros//`. diff --git a/tools/import_distro.py b/tools/import_distro.py index a5cbd1abd..cf1ea4967 100644 --- a/tools/import_distro.py +++ b/tools/import_distro.py @@ -35,21 +35,12 @@ from pathlib import Path import yaml -from ruamel.yaml import YAML import robostack as rs DERIVED_KEYS = ("conda_index", "patch_dir", "rosdistro_snapshot", "rosdistro_additional_recipes", "skip_existing") -def _ry() -> YAML: - ry = YAML() - ry.preserve_quotes = True - ry.width = 4096 - ry.indent(mapping=2, sequence=4, offset=2) - return ry - - def _norm(x): if isinstance(x, str): return x.replace("-", "_") @@ -125,7 +116,7 @@ def copy_dir(self, path: str, dest: Path) -> None: def import_distro(src: Source, distro: str | None) -> str: - ry = _ry() + ry = rs.ruamel_yaml() vinca = ry.load(src.text("vinca.yaml")) # the old repository's names (ros--*) follow its distribution, also when # --distro imports it under another name diff --git a/tools/maintenance.py b/tools/maintenance.py index 1028a013e..aa9b3bd3f 100644 --- a/tools/maintenance.py +++ b/tools/maintenance.py @@ -90,16 +90,6 @@ def fold(title: str, body: list[str]) -> None: return "\n".join(lines) -def _ruamel(): - from ruamel.yaml import YAML - - ry = YAML() - ry.preserve_quotes = True - ry.width = 4096 - ry.indent(mapping=2, sequence=4, offset=2) - return ry - - REPODATA_SUBDIRS = ("noarch", "linux-64", "linux-aarch64", "osx-64", "osx-arm64", "win-64") @@ -167,7 +157,7 @@ def bump_rebuild(distro: str) -> list[str]: lines = [f"`build_number` {current} → {number} ({source} + 1), mutex {old_mutex} → {new_mutex} (rebuilds every package)"] if info.is_file(): - ry = _ruamel() + ry = rs.ruamel_yaml() data = ry.load(info.read_text()) or {} dropped = [] for pkg in list(data): @@ -246,7 +236,7 @@ def bump_partial(distro: str, packages: dict[str, str]) -> list[str]: number = (current if released is None else max(current, released)) + 1 info = rs.DISTROS / distro / "pkg_additional_info.yaml" - ry = _ruamel() + ry = rs.ruamel_yaml() data = (ry.load(info.read_text()) if info.is_file() else None) or {} for pkg in packages: entry = data.get(pkg) @@ -709,7 +699,7 @@ def _available(distro: str) -> set[str]: def _append_selection(path: Path, packages: list[str]) -> None: """Append to packages_select_by_deps keeping the file's comments (round trip).""" - ry = _ruamel() + ry = rs.ruamel_yaml() data = ry.load(path.read_text()) if path.is_file() else {} data.setdefault("packages_select_by_deps", []) items = data["packages_select_by_deps"] @@ -874,7 +864,7 @@ def merge_build_numbers(path: Path, entries: dict[str, int]) -> list[str]: packages that changed.""" from vinca.sort_yaml_keys import sort_mapping_keys - ry = _ruamel() + ry = rs.ruamel_yaml() data = (ry.load(path.read_text()) if path.is_file() else None) or {} changed = [] for name, number in sorted(entries.items()): diff --git a/tools/robostack.py b/tools/robostack.py index b27fe05d7..8b723dfa9 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -20,6 +20,7 @@ from __future__ import annotations import argparse +import io import json import os import re @@ -208,45 +209,103 @@ def run_build(cmd: list[str], cwd: Path, show=None) -> int: def vinca_pinning(distro: str) -> str: """shared/pinning/conda_forge.yaml + overrides.yaml, adjusted by distro.yaml. - Shared override entries that the distribution overrides itself are left out - together with the comment lines above them (comments carry the conda-build - selectors, so this works on text, not on parsed YAML). + The files are read and written with ruamel.yaml's round trip, which keeps + comments, and with them the conda-build selectors (`- 2.39 # [riscv64]`). + A distribution's own pinning_overrides replace the shared entries of the same + name, together with the comment above them. The distribution's own comments + are left out: vinca-pinning-render moves its entries to their place in the + conda-forge pinning, and the comments would end up above other keys. """ - s = settings(distro) + from ruamel.yaml.comments import CommentedMap, CommentedSeq + + yaml = ruamel_yaml() + s = yaml.load((DISTROS / distro / "distro.yaml").read_text()) or {} own = s.get("pinning_overrides") or {} - out = [f"# Generated by `pixi run rs {distro} render-pinning` from shared/pinning/ and", - f"# distros/{distro}/distro.yaml; edit those instead.", ""] + shared = yaml.load((SHARED / "pinning" / "overrides.yaml").read_text())["pinning_overrides"] + shared_comments = key_comments(shared) + + out = CommentedMap() if s.get("conda_forge_pinning_version"): - out.append(f"conda_forge_pinning_version: {s['conda_forge_pinning_version']}") - migrations = s.get("conda_forge_migrations") or [] - out.append("migrations:" if migrations else "migrations: []") - out += [f" - {m}" for m in migrations] + out["conda_forge_pinning_version"] = s["conda_forge_pinning_version"] + out["migrations"] = s.get("conda_forge_migrations") or [] + cut_trailing_comments(out["migrations"]) else: - out += [l for l in (SHARED / "pinning" / "conda_forge.yaml").read_text().splitlines() - if not l.startswith("#")] - comments: list[str] = [] - skip = False - for line in (SHARED / "pinning" / "overrides.yaml").read_text().splitlines(): - if line.startswith("#"): - continue - if line.startswith(" #"): - comments.append(line) - elif line.startswith(" ") and not line.startswith(" "): - skip = line[2:].split(":")[0] in own - if not skip: - out += comments + [line] - comments = [] - elif line.startswith(" "): - if not skip: - out.append(line) - else: - out.append(line) + out.update(yaml.load((SHARED / "pinning" / "conda_forge.yaml").read_text())) + overrides = out["pinning_overrides"] = CommentedMap() + for key, value in shared.items(): + if key not in own: + overrides[key] = value + if key in shared_comments: + overrides.yaml_set_comment_before_after_key(key, before=shared_comments[key], indent=2) + for key, value in own.items(): + if not isinstance(value, list): + value = CommentedSeq([value]) + cut_trailing_comments(value) + value.fa.set_block_style() + overrides[key] = value if own: - out.append(f" # ros-{distro} specific (distros/{distro}/distro.yaml)") - for key, value in own.items(): - out.append(f" {key}:") - out += [f" - {v}" for v in (value if isinstance(value, list) else [value])] - return "\n".join(out) + "\n" + overrides.yaml_set_comment_before_after_key( + next(iter(own)), before=f"ros-{distro} specific (distros/{distro}/distro.yaml)", indent=2) + + out.yaml_set_start_comment(f"Generated by `pixi run rs {distro} render-pinning` from shared/pinning/ and\n" + f"distros/{distro}/distro.yaml; edit those instead.\n") + stream = io.StringIO() + yaml.dump(out, stream) + return stream.getvalue() + + +def ruamel_yaml(): + """ruamel.yaml round trip in the layout of the repository's YAML files.""" + import ruamel.yaml + + yaml = ruamel.yaml.YAML() + yaml.preserve_quotes = True + yaml.width = 4096 + yaml.indent(mapping=2, sequence=4, offset=2) + return yaml + + +def _comment_text(text: str) -> str: + return "\n".join(line.strip().removeprefix("#").removeprefix(" ") + for line in text.splitlines() if line.strip()) + + +def cut_trailing_comments(value) -> str: + """Cut the comment lines that follow a block list's last item (ruamel keeps + the comment above the next key there) and return them; the item's own + comment, e.g. its selector, stays.""" + token = value.ca.items.get(len(value) - 1, [None])[0] if isinstance(value, list) and value else None + if token is None: + return "" + own_line, _, rest = token.value.partition("\n") + if own_line.strip(): + token.value = own_line + "\n" + else: + value.ca.items[len(value) - 1][0] = None + return rest + + +def key_comments(mapping) -> dict[str, str]: + """The comment above each key of a ruamel mapping, without the "# ". + + ruamel attaches it to whatever precedes the key: the mapping (first key), the + end of the previous block list (cut from there, see cut_trailing_comments), + the previous key after a scalar value, or the key itself after a flow value.""" + comments = {} + previous = None + for key in mapping: + if previous is None: + text = "".join(t.value for t in (mapping.ca.comment or [None, []])[1] or []) + elif isinstance(mapping[previous], list) and mapping[previous]: + text = cut_trailing_comments(mapping[previous]) + else: + token = mapping.ca.items.get(previous, [None] * 3)[2] + text = token.value.partition("\n")[2] if token else "" + text += "".join(t.value for t in mapping.ca.items.get(key, [None, None])[1] or []) + if text := _comment_text(text): + comments[key] = text + previous = key + return comments def load_yaml(path: Path) -> dict: diff --git a/tools/tests/test_pinning_update.py b/tools/tests/test_pinning_update.py index 335f29b0c..18c4f1b4f 100644 --- a/tools/tests/test_pinning_update.py +++ b/tools/tests/test_pinning_update.py @@ -46,3 +46,58 @@ def test_pin_report_lists_the_direct_users(): assert "- `zig-compiler`: `(none)` → `zig`" in lines # nothing used: the folded list comes first assert maintenance.pin_report({"zig-compiler": (None, "zig")}, {})[0] == "
" + + +def test_vinca_pinning_keeps_selectors_and_replaces_shared_entries(monkeypatch, tmp_path): + (tmp_path / "pinning").mkdir() + (tmp_path / "pinning" / "conda_forge.yaml").write_text("# header\nconda_forge_pinning_version: '1'\nmigrations: []\n") + (tmp_path / "pinning" / "overrides.yaml").write_text( + "pinning_overrides:\n" + " # why channels are dropped\n" + " channel_sources: null\n" + " # why glibc is pinned\n" + " glibc:\n" + " - 2.28 # [not riscv64]\n" + " - 2.39 # [riscv64]\n" + " # why python is pinned\n" + " python:\n" + " - 3.14.*\n" + " # why lua is pinned\n" + " lua:\n" + " - 5.4\n" + ) + (tmp_path / "x").mkdir() + (tmp_path / "x" / "distro.yaml").write_text( + "pinning_overrides:\n" + " xtensor: ['0.25.0']\n" + " # not carried over\n" + " python:\n" + " - 3.12.* # [not riscv64]\n" + " - 3.14.*\n" + "# nor is this one\n" + "rosdistro_sync: x/2026-01-01\n" + ) + monkeypatch.setattr(rs, "SHARED", tmp_path) + monkeypatch.setattr(rs, "DISTROS", tmp_path) + assert rs.vinca_pinning("x") == ( + "# Generated by `pixi run rs x render-pinning` from shared/pinning/ and\n" + "# distros/x/distro.yaml; edit those instead.\n" + "conda_forge_pinning_version: '1'\n" + "migrations: []\n" + "pinning_overrides:\n" + " # why channels are dropped\n" + " channel_sources:\n" + " # why glibc is pinned\n" + " glibc:\n" + " - 2.28 # [not riscv64]\n" + " - 2.39 # [riscv64]\n" + " # why lua is pinned\n" + " lua:\n" + " - 5.4\n" + " # ros-x specific (distros/x/distro.yaml)\n" + " xtensor:\n" + " - '0.25.0'\n" + " python:\n" + " - 3.12.* # [not riscv64]\n" + " - 3.14.*\n" + )