render-pinning: keep conda-build selectors in distro.yaml pinning_overrides

README.md

@@ -81,7 +81,7 @@ combines the two, and the files next to each:
 - **`patch/*.patch` and `tests/`:** per package, the distribution's file replaces the shared one. In tests, `- if: ros_distro in ["humble", "jazzy"]` blocks select distribution-specific parts.
 - **`vinca_pinning.yaml`:** the shared pinning, adjusted by `distro.yaml`:
   - `conda_forge_pinning_version` and `conda_forge_migrations` keep a distribution on its own conda-forge pinning, for example until its next full rebuild;
-  - `pinning_overrides` replace shared pins by name, for example jazzy's Python 3.12.
+  - `pinning_overrides` replace shared pins by name, for example jazzy's Python 3.12. Items written as a block list keep their conda-build selectors (`- 3.12.*  # [not riscv64]`).
   - `pixi run rs <distro> render-pinning` turns this into the committed `conda_build_config.yaml`.
 
 A fix that applies to every distribution goes into `shared/`, a version-specific one into `distros/<distro>/`.

tools/import_distro.py

@@ -35,21 +35,12 @@
 from pathlib import Path
 
 import yaml
-from ruamel.yaml import YAML
 
 import robostack as rs
 
 DERIVED_KEYS = ("conda_index", "patch_dir", "rosdistro_snapshot", "rosdistro_additional_recipes", "skip_existing")
 
 
-def _ry() -> YAML:
-    ry = YAML()
-    ry.preserve_quotes = True
-    ry.width = 4096
-    ry.indent(mapping=2, sequence=4, offset=2)
-    return ry
-
-
 def _norm(x):
     if isinstance(x, str):
         return x.replace("-", "_")
@@ -125,7 +116,7 @@ def copy_dir(self, path: str, dest: Path) -> None:
 
 
 def import_distro(src: Source, distro: str | None) -> str:
-    ry = _ry()
+    ry = rs.ruamel_yaml()
     vinca = ry.load(src.text("vinca.yaml"))
     # the old repository's names (ros-<source>-*) follow its distribution, also when
     # --distro imports it under another name

tools/maintenance.py

@@ -90,16 +90,6 @@ def fold(title: str, body: list[str]) -> None:
     return "\n".join(lines)
 
 
-def _ruamel():
-    from ruamel.yaml import YAML
-
-    ry = YAML()
-    ry.preserve_quotes = True
-    ry.width = 4096
-    ry.indent(mapping=2, sequence=4, offset=2)
-    return ry
-
-
 REPODATA_SUBDIRS = ("noarch", "linux-64", "linux-aarch64", "osx-64", "osx-arm64", "win-64")
 
 
@@ -167,7 +157,7 @@ def bump_rebuild(distro: str) -> list[str]:
     lines = [f"`build_number` {current} → {number} ({source} + 1), mutex {old_mutex} → {new_mutex} (rebuilds every package)"]
 
     if info.is_file():
-        ry = _ruamel()
+        ry = rs.ruamel_yaml()
         data = ry.load(info.read_text()) or {}
         dropped = []
         for pkg in list(data):
@@ -246,7 +236,7 @@ def bump_partial(distro: str, packages: dict[str, str]) -> list[str]:
     number = (current if released is None else max(current, released)) + 1
 
     info = rs.DISTROS / distro / "pkg_additional_info.yaml"
-    ry = _ruamel()
+    ry = rs.ruamel_yaml()
     data = (ry.load(info.read_text()) if info.is_file() else None) or {}
     for pkg in packages:
         entry = data.get(pkg)
@@ -709,7 +699,7 @@ def _available(distro: str) -> set[str]:
 
 def _append_selection(path: Path, packages: list[str]) -> None:
     """Append to packages_select_by_deps keeping the file's comments (round trip)."""
-    ry = _ruamel()
+    ry = rs.ruamel_yaml()
     data = ry.load(path.read_text()) if path.is_file() else {}
     data.setdefault("packages_select_by_deps", [])
     items = data["packages_select_by_deps"]
@@ -874,7 +864,7 @@ def merge_build_numbers(path: Path, entries: dict[str, int]) -> list[str]:
     packages that changed."""
     from vinca.sort_yaml_keys import sort_mapping_keys
 
-    ry = _ruamel()
+    ry = rs.ruamel_yaml()
     data = (ry.load(path.read_text()) if path.is_file() else None) or {}
     changed = []
     for name, number in sorted(entries.items()):

tools/robostack.py

@@ -20,6 +20,7 @@
 from __future__ import annotations
 
 import argparse
+import io
 import json
 import os
 import re
@@ -208,45 +209,103 @@ def run_build(cmd: list[str], cwd: Path, show=None) -> int:
 def vinca_pinning(distro: str) -> str:
     """shared/pinning/conda_forge.yaml + overrides.yaml, adjusted by distro.yaml.
 
-    Shared override entries that the distribution overrides itself are left out
-    together with the comment lines above them (comments carry the conda-build
-    selectors, so this works on text, not on parsed YAML).
+    The files are read and written with ruamel.yaml's round trip, which keeps
+    comments, and with them the conda-build selectors (`- 2.39  # [riscv64]`).
+    A distribution's own pinning_overrides replace the shared entries of the same
+    name, together with the comment above them. The distribution's own comments
+    are left out: vinca-pinning-render moves its entries to their place in the
+    conda-forge pinning, and the comments would end up above other keys.
     """
-    s = settings(distro)
+    from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
+    yaml = ruamel_yaml()
+    s = yaml.load((DISTROS / distro / "distro.yaml").read_text()) or {}
     own = s.get("pinning_overrides") or {}
-    out = [f"# Generated by `pixi run rs {distro} render-pinning` from shared/pinning/ and",
-           f"# distros/{distro}/distro.yaml; edit those instead.", ""]
+    shared = yaml.load((SHARED / "pinning" / "overrides.yaml").read_text())["pinning_overrides"]
+    shared_comments = key_comments(shared)
+
+    out = CommentedMap()
     if s.get("conda_forge_pinning_version"):
-        out.append(f"conda_forge_pinning_version: {s['conda_forge_pinning_version']}")
-        migrations = s.get("conda_forge_migrations") or []
-        out.append("migrations:" if migrations else "migrations: []")
-        out += [f"  - {m}" for m in migrations]
+        out["conda_forge_pinning_version"] = s["conda_forge_pinning_version"]
+        out["migrations"] = s.get("conda_forge_migrations") or []
+        cut_trailing_comments(out["migrations"])
     else:
-        out += [l for l in (SHARED / "pinning" / "conda_forge.yaml").read_text().splitlines()
-                if not l.startswith("#")]
-    comments: list[str] = []
-    skip = False
-    for line in (SHARED / "pinning" / "overrides.yaml").read_text().splitlines():
-        if line.startswith("#"):
-            continue
-        if line.startswith("  #"):
-            comments.append(line)
-        elif line.startswith("  ") and not line.startswith("   "):
-            skip = line[2:].split(":")[0] in own
-            if not skip:
-                out += comments + [line]
-            comments = []
-        elif line.startswith("    "):
-            if not skip:
-                out.append(line)
-        else:
-            out.append(line)
+        out.update(yaml.load((SHARED / "pinning" / "conda_forge.yaml").read_text()))
+    overrides = out["pinning_overrides"] = CommentedMap()
+    for key, value in shared.items():
+        if key not in own:
+            overrides[key] = value
+            if key in shared_comments:
+                overrides.yaml_set_comment_before_after_key(key, before=shared_comments[key], indent=2)
+    for key, value in own.items():
+        if not isinstance(value, list):
+            value = CommentedSeq([value])
+        cut_trailing_comments(value)
+        value.fa.set_block_style()
+        overrides[key] = value
     if own:
-        out.append(f"  # ros-{distro} specific (distros/{distro}/distro.yaml)")
-        for key, value in own.items():
-            out.append(f"  {key}:")
-            out += [f"    - {v}" for v in (value if isinstance(value, list) else [value])]
-    return "\n".join(out) + "\n"
+        overrides.yaml_set_comment_before_after_key(
+            next(iter(own)), before=f"ros-{distro} specific (distros/{distro}/distro.yaml)", indent=2)
+
+    out.yaml_set_start_comment(f"Generated by `pixi run rs {distro} render-pinning` from shared/pinning/ and\n"
+                               f"distros/{distro}/distro.yaml; edit those instead.\n")
+    stream = io.StringIO()
+    yaml.dump(out, stream)
+    return stream.getvalue()
+
+
+def ruamel_yaml():
+    """ruamel.yaml round trip in the layout of the repository's YAML files."""
+    import ruamel.yaml
+
+    yaml = ruamel.yaml.YAML()
+    yaml.preserve_quotes = True
+    yaml.width = 4096
+    yaml.indent(mapping=2, sequence=4, offset=2)
+    return yaml
+
+
+def _comment_text(text: str) -> str:
+    return "\n".join(line.strip().removeprefix("#").removeprefix(" ")
+                     for line in text.splitlines() if line.strip())
+
+
+def cut_trailing_comments(value) -> str:
+    """Cut the comment lines that follow a block list's last item (ruamel keeps
+    the comment above the next key there) and return them; the item's own
+    comment, e.g. its selector, stays."""
+    token = value.ca.items.get(len(value) - 1, [None])[0] if isinstance(value, list) and value else None
+    if token is None:
+        return ""
+    own_line, _, rest = token.value.partition("\n")
+    if own_line.strip():
+        token.value = own_line + "\n"
+    else:
+        value.ca.items[len(value) - 1][0] = None
+    return rest
+
+
+def key_comments(mapping) -> dict[str, str]:
+    """The comment above each key of a ruamel mapping, without the "# ".
+
+    ruamel attaches it to whatever precedes the key: the mapping (first key), the
+    end of the previous block list (cut from there, see cut_trailing_comments),
+    the previous key after a scalar value, or the key itself after a flow value."""
+    comments = {}
+    previous = None
+    for key in mapping:
+        if previous is None:
+            text = "".join(t.value for t in (mapping.ca.comment or [None, []])[1] or [])
+        elif isinstance(mapping[previous], list) and mapping[previous]:
+            text = cut_trailing_comments(mapping[previous])
+        else:
+            token = mapping.ca.items.get(previous, [None] * 3)[2]
+            text = token.value.partition("\n")[2] if token else ""
+        text += "".join(t.value for t in mapping.ca.items.get(key, [None, None])[1] or [])
+        if text := _comment_text(text):
+            comments[key] = text
+        previous = key
+    return comments
 
 
 def load_yaml(path: Path) -> dict:

tools/tests/test_pinning_update.py

@@ -46,3 +46,58 @@ def test_pin_report_lists_the_direct_users():
     assert "- `zig-compiler`: `(none)` → `zig`" in lines
     # nothing used: the folded list comes first
     assert maintenance.pin_report({"zig-compiler": (None, "zig")}, {})[0] == "<details>"
+
+
+def test_vinca_pinning_keeps_selectors_and_replaces_shared_entries(monkeypatch, tmp_path):
+    (tmp_path / "pinning").mkdir()
+    (tmp_path / "pinning" / "conda_forge.yaml").write_text("# header\nconda_forge_pinning_version: '1'\nmigrations: []\n")
+    (tmp_path / "pinning" / "overrides.yaml").write_text(
+        "pinning_overrides:\n"
+        "  # why channels are dropped\n"
+        "  channel_sources: null\n"
+        "  # why glibc is pinned\n"
+        "  glibc:\n"
+        "    - 2.28   # [not riscv64]\n"
+        "    - 2.39   # [riscv64]\n"
+        "  # why python is pinned\n"
+        "  python:\n"
+        "    - 3.14.*\n"
+        "  # why lua is pinned\n"
+        "  lua:\n"
+        "    - 5.4\n"
+    )
+    (tmp_path / "x").mkdir()
+    (tmp_path / "x" / "distro.yaml").write_text(
+        "pinning_overrides:\n"
+        "  xtensor: ['0.25.0']\n"
+        "  # not carried over\n"
+        "  python:\n"
+        "    - 3.12.*   # [not riscv64]\n"
+        "    - 3.14.*\n"
+        "# nor is this one\n"
+        "rosdistro_sync: x/2026-01-01\n"
+    )
+    monkeypatch.setattr(rs, "SHARED", tmp_path)
+    monkeypatch.setattr(rs, "DISTROS", tmp_path)
+    assert rs.vinca_pinning("x") == (
+        "# Generated by `pixi run rs x render-pinning` from shared/pinning/ and\n"
+        "# distros/x/distro.yaml; edit those instead.\n"
+        "conda_forge_pinning_version: '1'\n"
+        "migrations: []\n"
+        "pinning_overrides:\n"
+        "  # why channels are dropped\n"
+        "  channel_sources:\n"
+        "  # why glibc is pinned\n"
+        "  glibc:\n"
+        "    - 2.28   # [not riscv64]\n"
+        "    - 2.39   # [riscv64]\n"
+        "  # why lua is pinned\n"
+        "  lua:\n"
+        "    - 5.4\n"
+        "  # ros-x specific (distros/x/distro.yaml)\n"
+        "  xtensor:\n"
+        "    - '0.25.0'\n"
+        "  python:\n"
+        "    - 3.12.*   # [not riscv64]\n"
+        "    - 3.14.*\n"
+    )