From 33fe55fc4052b8a7ad3d9e88c1eca91ecf672d9a Mon Sep 17 00:00:00 2001 From: Micah Smith Date: Mon, 12 Oct 2026 17:20:49 -0400 Subject: [PATCH] test(autoland): pin step execution behavior with golden scenarios MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Phase 0 of the autoland step-abstraction stack: before `execute_plan` and its step rendering/persistence are refactored (an `AutolandStep` base class, `StepView`, injected services, plan optimization, cancellation), this pins what a user can observe of a whole autoland run today, so those refactors can show they change nothing. Each scenario runs the real `stack-pr autoland` flow (`run_autoland` → fresh/resume/replan → `engine.execute_plan`) against a scripted GitHub and local clone, and compares five artifacts with golden files in `tests/golden/autoland/`: - `.out.txt`: the full console output (plain console, rich off), including what was typed at prompts - `.checkpoint.json`: the last checkpoint the run saved - `.status.txt` / `.status.json`: `autoland --status` and `--status -o json` over that checkpoint - `.plan.txt`: that plan as the `-i` editor shows it; the test also checks it parses back (`parse_plan`) to the same text Scenarios: 1. `one_at_a_time`: two PRs landed one at a time, with a workflow and a confirmation between them (checks pending at first, a review that comes in later, one wrong answer at the prompt) 2. `merge_as_stack`: a run of two `l` steps merged as a GitHub stack, then a workflow, a bare `c`, and a lone `l` 3. `stack_merge_falls_through`: the stack merge fails after landing only its bottom PR; the rest lands one at a time 4. `checks_fail`: checks still fail after their rerun, so the run aborts 5. `workflow_fails`: every deploy run fails, so the workflow checkpoint times out and the run aborts 6. `confirm_eof`: a confirmation with no terminal (EOF), so the run aborts 7. `resume_mid_plan`: `--resume` after a workflow timeout, in a plan with an already-landed prefix (`l 101`, `l 102`) and assumed-completed workflow/confirm steps 8. `landed_prefix`: a fresh run of a plan whose first PRs already landed 9. `replan_carry_over`: `--replan` with an edited plan file; finished checkpoints carry over, a new one runs, and a removed one is reported as not carried over All scenario setup is in `tests/autoland_scenarios.py`, and every patch is in its `_install_seams`, so when the next commits move a seam, only that function needs re-pointing. The patches: `runtime.console` (the plain console, with scripted `input()`), `runtime.HAVE_RICH`, `display.GLYPHS`, `runtime.resilient_sleep` (advances the fake's clock instead of sleeping), and `shell_commands.run_shell_command` / `cli.get_command_output`, which a `FakeWorld` answers for `gh` and `git`. It also stubs `cli.get_stack` and `cli.command_submit`, pins `git.get_current_branch_name`, and makes any real subprocess fail the test. Because the fake sits at the subprocess boundary, the real `gh.GitHub` client, the retry wrapper, and its `$ gh ...` echo all run. The golden artifacts come from real output and the real checkpoint file, so they don't depend on how anything is patched. Temp paths become `` and save times ``; SHAs are deterministic. To regenerate after an intended behavior change (then review the diff): ```bash UPDATE_GOLDEN=1 uv run --no-sync pytest tests/test_autoland_golden.py ``` No `src/` changes. ## Test plan - [x] New `tests/test_autoland_golden.py`: 9 golden scenarios, about 0.2s total, no real sleeps or subprocesses - [x] Regenerating twice gives byte-identical golden files (deterministic) - [x] Passes with gh unauthenticated (`GH_CONFIG_DIR=$(mktemp -d) GH_TOKEN=`) - [x] Editing a golden file makes its scenario fail, with a message pointing at `UPDATE_GOLDEN=1` - [x] Reviewed every golden output by hand; it reads like real autoland output 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 5.5 (1M context) stack-info: PR: https://github.com/micahjsmith/stack-pr/pull/68, branch: micahjsmith/stack/2 --- tests/autoland_scenarios.py | 521 ++++++++++++++++++ tests/golden/autoland/README.md | 28 + .../autoland/checks_fail.checkpoint.json | 45 ++ tests/golden/autoland/checks_fail.out.txt | 33 ++ tests/golden/autoland/checks_fail.plan.txt | 13 + tests/golden/autoland/checks_fail.status.json | 42 ++ tests/golden/autoland/checks_fail.status.txt | 18 + .../autoland/confirm_eof.checkpoint.json | 50 ++ tests/golden/autoland/confirm_eof.out.txt | 52 ++ tests/golden/autoland/confirm_eof.plan.txt | 14 + tests/golden/autoland/confirm_eof.status.json | 51 ++ tests/golden/autoland/confirm_eof.status.txt | 18 + .../autoland/landed_prefix.checkpoint.json | 56 ++ tests/golden/autoland/landed_prefix.out.txt | 53 ++ tests/golden/autoland/landed_prefix.plan.txt | 17 + .../golden/autoland/landed_prefix.status.json | 80 +++ .../golden/autoland/landed_prefix.status.txt | 19 + .../autoland/merge_as_stack.checkpoint.json | 72 +++ tests/golden/autoland/merge_as_stack.out.txt | 78 +++ tests/golden/autoland/merge_as_stack.plan.txt | 16 + .../autoland/merge_as_stack.status.json | 71 +++ .../golden/autoland/merge_as_stack.status.txt | 18 + .../autoland/one_at_a_time.checkpoint.json | 56 ++ tests/golden/autoland/one_at_a_time.out.txt | 78 +++ tests/golden/autoland/one_at_a_time.plan.txt | 15 + .../golden/autoland/one_at_a_time.status.json | 60 ++ .../golden/autoland/one_at_a_time.status.txt | 17 + .../replan_carry_over.checkpoint.json | 66 +++ .../golden/autoland/replan_carry_over.out.txt | 65 +++ .../autoland/replan_carry_over.plan.txt | 19 + .../autoland/replan_carry_over.status.json | 98 ++++ .../autoland/replan_carry_over.status.txt | 21 + .../autoland/resume_mid_plan.checkpoint.json | 56 ++ tests/golden/autoland/resume_mid_plan.out.txt | 50 ++ .../golden/autoland/resume_mid_plan.plan.txt | 17 + .../autoland/resume_mid_plan.status.json | 80 +++ .../autoland/resume_mid_plan.status.txt | 19 + .../stack_merge_falls_through.checkpoint.json | 61 ++ .../stack_merge_falls_through.out.txt | 65 +++ .../stack_merge_falls_through.plan.txt | 14 + .../stack_merge_falls_through.status.json | 53 ++ .../stack_merge_falls_through.status.txt | 16 + .../autoland/workflow_fails.checkpoint.json | 51 ++ tests/golden/autoland/workflow_fails.out.txt | 56 ++ tests/golden/autoland/workflow_fails.plan.txt | 14 + .../autoland/workflow_fails.status.json | 51 ++ .../golden/autoland/workflow_fails.status.txt | 19 + tests/test_autoland_golden.py | 269 +++++++++ 48 files changed, 2751 insertions(+) create mode 100644 tests/autoland_scenarios.py create mode 100644 tests/golden/autoland/README.md create mode 100644 tests/golden/autoland/checks_fail.checkpoint.json create mode 100644 tests/golden/autoland/checks_fail.out.txt create mode 100644 tests/golden/autoland/checks_fail.plan.txt create mode 100644 tests/golden/autoland/checks_fail.status.json create mode 100644 tests/golden/autoland/checks_fail.status.txt create mode 100644 tests/golden/autoland/confirm_eof.checkpoint.json create mode 100644 tests/golden/autoland/confirm_eof.out.txt create mode 100644 tests/golden/autoland/confirm_eof.plan.txt create mode 100644 tests/golden/autoland/confirm_eof.status.json create mode 100644 tests/golden/autoland/confirm_eof.status.txt create mode 100644 tests/golden/autoland/landed_prefix.checkpoint.json create mode 100644 tests/golden/autoland/landed_prefix.out.txt create mode 100644 tests/golden/autoland/landed_prefix.plan.txt create mode 100644 tests/golden/autoland/landed_prefix.status.json create mode 100644 tests/golden/autoland/landed_prefix.status.txt create mode 100644 tests/golden/autoland/merge_as_stack.checkpoint.json create mode 100644 tests/golden/autoland/merge_as_stack.out.txt create mode 100644 tests/golden/autoland/merge_as_stack.plan.txt create mode 100644 tests/golden/autoland/merge_as_stack.status.json create mode 100644 tests/golden/autoland/merge_as_stack.status.txt create mode 100644 tests/golden/autoland/one_at_a_time.checkpoint.json create mode 100644 tests/golden/autoland/one_at_a_time.out.txt create mode 100644 tests/golden/autoland/one_at_a_time.plan.txt create mode 100644 tests/golden/autoland/one_at_a_time.status.json create mode 100644 tests/golden/autoland/one_at_a_time.status.txt create mode 100644 tests/golden/autoland/replan_carry_over.checkpoint.json create mode 100644 tests/golden/autoland/replan_carry_over.out.txt create mode 100644 tests/golden/autoland/replan_carry_over.plan.txt create mode 100644 tests/golden/autoland/replan_carry_over.status.json create mode 100644 tests/golden/autoland/replan_carry_over.status.txt create mode 100644 tests/golden/autoland/resume_mid_plan.checkpoint.json create mode 100644 tests/golden/autoland/resume_mid_plan.out.txt create mode 100644 tests/golden/autoland/resume_mid_plan.plan.txt create mode 100644 tests/golden/autoland/resume_mid_plan.status.json create mode 100644 tests/golden/autoland/resume_mid_plan.status.txt create mode 100644 tests/golden/autoland/stack_merge_falls_through.checkpoint.json create mode 100644 tests/golden/autoland/stack_merge_falls_through.out.txt create mode 100644 tests/golden/autoland/stack_merge_falls_through.plan.txt create mode 100644 tests/golden/autoland/stack_merge_falls_through.status.json create mode 100644 tests/golden/autoland/stack_merge_falls_through.status.txt create mode 100644 tests/golden/autoland/workflow_fails.checkpoint.json create mode 100644 tests/golden/autoland/workflow_fails.out.txt create mode 100644 tests/golden/autoland/workflow_fails.plan.txt create mode 100644 tests/golden/autoland/workflow_fails.status.json create mode 100644 tests/golden/autoland/workflow_fails.status.txt create mode 100644 tests/test_autoland_golden.py diff --git a/tests/autoland_scenarios.py b/tests/autoland_scenarios.py new file mode 100644 index 0000000..03c2c0a --- /dev/null +++ b/tests/autoland_scenarios.py @@ -0,0 +1,521 @@ +"""Scripted end-to-end autoland runs, for the golden-file characterization tests. + +A scenario describes a stack, the GitHub state it lands against, a plan, and +the answers typed at the console. ``run_scenario`` drives the real +``stack-pr autoland`` flow (``run_autoland``, and through it +``engine.execute_plan``) over it and returns everything a user could observe: +the console output, the last checkpoint written, ``--status`` in both formats, +and the plan as the ``-i`` editor would show it. + +Every patch lives in ``_install_seams``, so a refactor that moves a seam only +has to re-point it there. The artifacts never depend on how things are patched: +they are rendered from the real console output and the real checkpoint file. + +The patched seams: + +- the console (``runtime.console``) is the plain-text console, with rich off, + the Unicode glyphs pinned, and ``input()`` scripted; +- ``runtime.resilient_sleep`` doesn't sleep, but advances ``FakeWorld``'s clock; +- subprocesses: ``shell_commands.run_shell_command`` (under ``runtime.run`` + and the ``gh`` JSON helpers) and ``cli.get_command_output`` are answered by + ``FakeWorld``, which plays both GitHub (``gh``) and the local clone (``git``); +- stack-pr itself: ``cli.get_stack`` returns the scenario's stack, + ``cli.command_submit`` (the resubmit after a merge) does nothing, and + ``git.get_current_branch_name`` is pinned; +- the SIGINT handler isn't installed, and any real subprocess fails the test. +""" + +from __future__ import annotations + +import builtins +import configparser +import copy +import hashlib +import json +import re +import subprocess +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import pytest +from pytest_mock import MockerFixture + +from stack_pr import cli, shell_commands +from stack_pr.autoland import commands, display, runtime +from stack_pr.autoland.display import _UNICODE_GLYPHS +from stack_pr.autoland.model import LandingContext, PRState, StackEntry +from stack_pr.autoland.plan import format_plan_for_editor, parse_plan +from stack_pr.autoland.runtime import _PlainConsole +from stack_pr.autoland.state import AutolandCheckpointer +from tests.helpers import PR_URL, common_args, mock_entry + +BRANCH = "feat" +OWNER, REPO = "o", "r" +WORKFLOW_RUN_ID = 9000 # + the PR number: the CI run a PR's checks belong to + + +def sha(label: str) -> str: + """A deterministic, realistic-looking commit SHA.""" + return hashlib.sha1(label.encode()).hexdigest() # noqa: S324 - not security + + +def commit_sha(pr: int) -> str: + """The local (and pushed) commit of PR *pr*.""" + return sha(f"commit-{pr}") + + +# --------------------------------------------------------------------------- +# The fake world: GitHub, and the local clone +# --------------------------------------------------------------------------- + + +@dataclass +class FakePR: + number: int + title: str + state: str = "OPEN" # OPEN, MERGED, CLOSED + review: str = "APPROVED" + # Clock ticks until a pending review turns into an approval (0: never). + approve_after: int = 0 + # The checks' bucket on each successive read (pass/fail/pending); the last + # one repeats. + checks: list[str] = field(default_factory=lambda: ["pass"]) + merge_state: str = "CLEAN" + # Clock ticks a PR spends in the merge queue before it merges. + queue_ticks: int = 1 + queued_for: int | None = None # ticks left in the queue; None: not queued + merge_commit: str = "" + + @property + def branch(self) -> str: + return f"TestBot/stack/{self.number}" + + def view(self, fields: str) -> dict[str, Any]: + values: dict[str, Any] = { + "number": self.number, + "title": self.title, + "state": self.state, + "reviewDecision": self.review, + "mergeStateStatus": self.merge_state, + "mergeable": "MERGEABLE", + "mergeCommit": {"oid": self.merge_commit} if self.merge_commit else None, + } + return {f: values[f] for f in fields.split(",")} + + +@dataclass +class FakeWorld: + """GitHub and the local clone, answering the commands autoland runs. + + Time only moves when autoland sleeps between polls (``tick``): that is when + queued PRs merge, pending reviews come in, and workflow runs finish. + """ + + prs: dict[int, FakePR] + # Workflow name -> the conclusion its runs reach. + workflows: dict[str, str] = field(default_factory=dict) + # How a stack merge request goes: "merge" (the whole stack is queued and + # merges), or "fail" (only stack_merge_lands merge, then GitHub reports + # stack_merge_message as the failure). + stack_merge: str = "merge" + stack_merge_lands: list[int] = field(default_factory=list) + stack_merge_message: str = "" + + main_head: str = field(default_factory=lambda: sha("main-0")) + stacks: dict[int, list[int]] = field(default_factory=dict) + next_stack_number: int = 7 + merge_requests: dict[str, list[int]] = field(default_factory=dict) + # Workflow name -> its runs, newest first. + runs: dict[str, list[dict[str, str]]] = field(default_factory=dict) + ticks: int = 0 + + def merge(self, pr: FakePR) -> None: + pr.state = "MERGED" + pr.queued_for = None + pr.merge_commit = sha(f"merge-{pr.number}") + self.main_head = pr.merge_commit + + def tick(self) -> None: + self.ticks += 1 + for pr in self.prs.values(): + if pr.queued_for is not None: + pr.queued_for -= 1 + if pr.queued_for <= 0: + self.merge(pr) + if pr.approve_after and self.ticks >= pr.approve_after: + pr.review = "APPROVED" + for name, runs in self.runs.items(): + for run in runs: + if run["status"] != "completed": + run["status"] = "completed" + run["conclusion"] = self.workflows[name] + + # --- command dispatch --------------------------------------------------- + + def __call__(self, cmd: list[str], stdin: str | None) -> str: + """Run *cmd*; return its stdout, or raise CalledProcessError.""" + if cmd[0] == "git": + return self._git(cmd[1:]) + if cmd[0] == "gh": + return self._gh(cmd[1:], stdin) + raise AssertionError(f"unexpected command: {cmd}") + + def _git(self, args: list[str]) -> str: + if args[0] in ("fetch", "rebase"): + return "" + if args == ["rev-parse", "origin/main"]: + return self.main_head + "\n" + if args[:3] == ["rev-parse", "--verify", "--quiet"]: + number = int(args[3].rsplit("/", 1)[-1]) + return commit_sha(number) + "\n" + if args[0] == "merge-base": + return self.main_head + "\n" + raise AssertionError(f"unexpected git command: {args}") + + def _gh(self, args: list[str], stdin: str | None) -> str: # noqa: PLR0911 + sub = args[:2] + if sub == ["repo", "view"]: + return json.dumps({"owner": {"login": OWNER}, "name": REPO}) + if sub == ["pr", "view"]: + return json.dumps(self.prs[int(args[2])].view(args[4])) + if sub == ["pr", "checks"]: + return json.dumps(self._checks(self.prs[int(args[2])])) + if sub == ["pr", "merge"]: + pr = self.prs[int(args[2])] + pr.queued_for = pr.queue_ticks + return "" + if sub == ["pr", "edit"]: + return "" + if sub == ["run", "rerun"]: + return "" + if sub == ["run", "list"]: + return json.dumps(self._workflow_runs(args[args.index("--workflow") + 1])) + if sub == ["api", "graphql"]: + return json.dumps(self._graphql(args)) + if sub[0] == "api" and args[1] == "-X": + body = json.loads(stdin) if stdin else None + path = args[3].removeprefix(f"repos/{OWNER}/{REPO}/") + out = self._rest(args[2], path, body) + return "" if out is None else json.dumps(out) + raise AssertionError(f"unexpected gh command: {args}") + + def _checks(self, pr: FakePR) -> list[dict[str, str]]: + bucket = pr.checks.pop(0) if len(pr.checks) > 1 else pr.checks[0] + state = {"pass": "SUCCESS", "fail": "FAILURE", "pending": "IN_PROGRESS"} + run = WORKFLOW_RUN_ID + pr.number + return [ + { + "name": "ci/test", + "state": state[bucket], + "bucket": bucket, + "link": f"https://github.com/{OWNER}/{REPO}/actions/runs/{run}/job/1", + "workflow": "CI", + } + ] + + def _workflow_runs(self, name: str) -> list[dict[str, str]]: + # A push to main starts a run of every workflow on it. + runs = self.runs.setdefault(name, []) + if not runs or runs[0]["headSha"] != self.main_head: + runs.insert( + 0, + {"headSha": self.main_head, "status": "in_progress", "conclusion": ""}, + ) + return [dict(r) for r in runs[:10]] + + def _graphql(self, args: list[str]) -> dict: + params = dict(a.split("=", 1) for a in args[3::2]) + if "number" in params: + queued = self.prs[int(params["number"])].queued_for is not None + entry = {"id": "MQE_1", "state": "QUEUED"} if queued else None + return {"data": {"repository": {"pullRequest": {"mergeQueueEntry": entry}}}} + return {"data": {"repository": {"mergeQueue": {"id": "MQ_1"}}}} + + def _stack_json(self, number: int) -> dict: + return { + "number": number, + "pull_requests": [ + {"number": n, "state": self.prs[n].state.lower()} + for n in self.stacks[number] + ], + } + + def _rest(self, method: str, path: str, body: dict | None) -> Any: + if method == "GET" and path.startswith("stacks?pull_request="): + pr = int(path.split("=", 1)[1]) + return [self._stack_json(n) for n, prs in self.stacks.items() if pr in prs] + if method == "POST" and path == "stacks": + assert body is not None + number = self.next_stack_number + self.next_stack_number += 1 + self.stacks[number] = list(body["pull_requests"]) + return self._stack_json(number) + if method == "POST" and path.endswith("/unstack"): + self.stacks.pop(int(path.split("/")[1])) + return None + if method == "PUT" and path.endswith("/merge-async"): + return self._merge_async(int(path.split("/")[1])) + if method == "GET" and "/merge-async/" in path: + prs = self.merge_requests[path.rsplit("/", 1)[1]] + if self.stack_merge == "fail": + return { + "status": "failed", + "details": {"message": self.stack_merge_message}, + } + merged = all(self.prs[n].state == "MERGED" for n in prs) + return {"status": "merged" if merged else "enqueued"} + raise AssertionError(f"unexpected gh api call: {method} {path}") + + def _merge_async(self, top: int) -> dict: + native = next(prs for prs in self.stacks.values() if top in prs) + prs = [ + n for n in native[: native.index(top) + 1] if self.prs[n].state == "OPEN" + ] + uuid = f"merge-request-{len(self.merge_requests) + 1}" + self.merge_requests[uuid] = prs + if self.stack_merge == "fail": + for n in self.stack_merge_lands: + self.merge(self.prs[n]) + else: + for n in prs: + self.prs[n].queued_for = self.prs[n].queue_ticks + return {"uuid": uuid} + + +# --------------------------------------------------------------------------- +# Scenarios +# --------------------------------------------------------------------------- + + +CheckpointBuilder = Callable[ + ["FakeWorld", Path], tuple[AutolandCheckpointer, LandingContext] +] + + +@dataclass +class Scenario: + name: str + prs: list[FakePR] + # The PRs still in the local stack, bottom first; the others have landed. + stack: list[int] + # The plan file's text; None runs the default plan. + plan: str | None = None + # Extra autoland flags (the plan file and the state file are added). + flags: list[str] = field(default_factory=list) + # [autoland] config, on top of the defaults below. + config: dict[str, str] = field(default_factory=dict) + # What the user types at each prompt; input after that is EOF. + answers: list[str] = field(default_factory=list) + workflows: dict[str, str] = field(default_factory=dict) + stack_merge: str = "merge" + stack_merge_lands: list[int] = field(default_factory=list) + stack_merge_message: str = "" + # PRs that landed before this run (and so are no longer in the stack). + landed: list[int] = field(default_factory=list) + # Builds the checkpoint of an earlier run, which this run resumes or + # replans. Called with the world and the plan file's path. + checkpoint: CheckpointBuilder | None = None + expect_exit: int | None = None + + +DEFAULT_CONFIG = { + "merge_queue": "true", + "poll_interval": "120", + "max_check_retries": "1", + "max_queue_retries": "1", + "workflow_timeout": "600", + "merge_as_stack": "false", +} + + +def stack_entries(world: FakeWorld, prs: list[int]) -> list[StackEntry]: + """The checkpointed stack entries for *prs*, as discovery would build them.""" + return [ + StackEntry( + pr_url=PR_URL.format(n), + pr_number=n, + branch=world.prs[n].branch, + title=world.prs[n].title, + review_decision=world.prs[n].review, + state=PRState.MERGED if world.prs[n].state == "MERGED" else PRState.PENDING, + ) + for n in prs + ] + + +@dataclass +class ScenarioResult: + """What a user could observe of a run, normalized for golden files.""" + + out: str # the console output of the run + checkpoint: str # the last checkpoint the run wrote + status_text: str # `autoland --status`, over that checkpoint + status_json: str # `autoland --status -o json`, over that checkpoint + plan: str # the checkpointed plan, as the -i editor shows it + plan_round_trip: str # that text, parsed and formatted again + exit_code: int | None + + +def _normalize(text: str, tmp_path: Path) -> str: + for path in sorted({str(tmp_path.resolve()), str(tmp_path)}, key=len, reverse=True): + text = text.replace(path, "") + # --status's "Last saved" line, in text and JSON. + text = re.sub( + r"\d{4}-\d\d-\d\d \d\d:\d\d:\d\d \((just now|\d+m ago)\)", "", text + ) + return re.sub(r'"last_saved": "[^"]+"', '"last_saved": ""', text) + + +def _install_seams( + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + scenario: Scenario, + world: FakeWorld, + tmp_path: Path, +) -> None: + monkeypatch.setenv("HOME", str(tmp_path / "home")) + + # Output: the plain console, Unicode glyphs, scripted input. + mocker.patch.object(runtime, "console", _PlainConsole()) + mocker.patch.object(runtime, "HAVE_RICH", False) + mocker.patch.object(display, "GLYPHS", _UNICODE_GLYPHS) + answers = list(scenario.answers) + + def scripted_input(prompt: object = "") -> str: + print(prompt, end="") + if not answers: + print() + raise EOFError + answer = answers.pop(0) + print(answer) + return answer + + monkeypatch.setattr(builtins, "input", scripted_input) + + # Time passes only when autoland sleeps between polls. + def sleep(_seconds: int) -> float: + world.tick() + return 0.0 + + mocker.patch.object(runtime, "resilient_sleep", side_effect=sleep) + + # Subprocesses: git and gh go to the world; anything else is a test bug. + def run_shell_command(cmd: Any, **kwargs: Any) -> subprocess.CompletedProcess: + args = [str(c) for c in cmd] + try: + out = world(args, kwargs.get("input")) + except subprocess.CalledProcessError as e: + return subprocess.CompletedProcess(args, e.returncode, "", e.stderr) + return subprocess.CompletedProcess(args, 0, out, "") + + mocker.patch.object( + shell_commands, "run_shell_command", side_effect=run_shell_command + ) + mocker.patch.object( + cli, + "get_command_output", + side_effect=lambda cmd, **_kw: world(cmd, None).strip(), + ) + mocker.patch.object( + subprocess, "run", side_effect=AssertionError("unexpected subprocess") + ) + + # stack-pr itself. + raw_stack = [ + mock_entry( + n, + head=world.prs[n].branch, + title=world.prs[n].title, + commit_id=commit_sha(n), + ) + for n in scenario.stack + ] + mocker.patch.object(cli, "get_stack", return_value=raw_stack) + mocker.patch.object(cli, "command_submit") + mocker.patch("stack_pr.git.get_current_branch_name", return_value=BRANCH) + mocker.patch.object(commands.signal, "signal") + + +def _autoland(argv: list[str], config: configparser.ConfigParser) -> int | None: + """Run `stack-pr autoland `; return its exit code, or None.""" + args = cli.create_argparser(config).parse_args(["autoland", *argv]) + try: + commands.run_autoland(common_args(base=""), args, config) + except SystemExit as e: + return int(e.code or 0) + return None + + +def run_scenario( + scenario: Scenario, + *, + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + tmp_path: Path, +) -> ScenarioResult: + world = FakeWorld( + # Copies: the world changes its PRs, and a scenario may run again. + prs={pr.number: copy.deepcopy(pr) for pr in scenario.prs}, + workflows=dict(scenario.workflows), + stack_merge=scenario.stack_merge, + stack_merge_lands=scenario.stack_merge_lands, + stack_merge_message=scenario.stack_merge_message, + ) + for number in scenario.landed: + world.merge(world.prs[number]) + _install_seams(mocker, monkeypatch, scenario, world, tmp_path) + state_path = AutolandCheckpointer.default_path(BRANCH) + config = configparser.ConfigParser() + config["autoland"] = {**DEFAULT_CONFIG, **scenario.config} + + argv = list(scenario.flags) + plan_file = tmp_path / "plan.autoland-plan" + if scenario.plan is not None: + plan_file.write_text(scenario.plan) + if scenario.checkpoint is None: + argv += ["--plan-file", str(plan_file)] + if scenario.checkpoint is not None: + old_checkpointer, old = scenario.checkpoint(world, plan_file) + old_checkpointer.path = state_path + old_checkpointer.save(old) + + # Keep each checkpoint as written: a run that succeeds deletes its own. + saved: list[str] = [] + real_save = AutolandCheckpointer.save + + def save(self: AutolandCheckpointer, ctx: LandingContext) -> None: + real_save(self, ctx) + saved.append(self.path.read_text()) + + mocker.patch.object(AutolandCheckpointer, "save", autospec=True, side_effect=save) + + capsys.readouterr() + exit_code = _autoland(argv, config) + out = capsys.readouterr().out + assert saved, "the run wrote no checkpoint" + + # --status, as it would read the run's last checkpoint. + state_path.write_text(saved[-1]) + assert _autoland(["--status"], config) is None + status_text = capsys.readouterr().out + assert _autoland(["--status", "-o", "json"], config) is None + status_json = capsys.readouterr().out + + _checkpointer, ctx = AutolandCheckpointer.load(state_path) + plan_text = format_plan_for_editor(ctx.stack, ctx.plan) + reparsed = parse_plan( + plan_text, ctx.stack, pr_is_merged=lambda n: world.prs[n].state == "MERGED" + ) + + return ScenarioResult( + out=_normalize(out, tmp_path), + checkpoint=_normalize(saved[-1], tmp_path), + status_text=_normalize(status_text, tmp_path), + status_json=_normalize(status_json, tmp_path), + plan=plan_text, + plan_round_trip=format_plan_for_editor(ctx.stack, reparsed), + exit_code=exit_code, + ) diff --git a/tests/golden/autoland/README.md b/tests/golden/autoland/README.md new file mode 100644 index 0000000..fc6e006 --- /dev/null +++ b/tests/golden/autoland/README.md @@ -0,0 +1,28 @@ +# Autoland golden files + +Expected output of the end-to-end autoland scenarios in +`tests/test_autoland_golden.py`. Each scenario runs `stack-pr autoland` against +a scripted GitHub and local clone (`tests/autoland_scenarios.py`) and writes: + +| File | What it holds | +| ---------------------------- | --------------------------------------------------------------------- | +| `.out.txt` | Everything the run printed (plain console, no rich), including input | +| `.checkpoint.json` | The last checkpoint the run saved (a successful run deletes it after) | +| `.status.txt` | `stack-pr autoland --status` over that checkpoint | +| `.status.json` | `stack-pr autoland --status -o json` over that checkpoint | +| `.plan.txt` | That checkpoint's plan, as the `-i` editor would show it | + +Temporary paths are replaced with `` and the checkpoint's save time with +``; everything else (SHAs included) is deterministic. + +## Regenerating + +When a change to autoland's behavior is intended, regenerate the files and +review the diff like any other change: + +```bash +UPDATE_GOLDEN=1 uv run --no-sync pytest tests/test_autoland_golden.py +git diff tests/golden/autoland +``` + +A refactor that is meant to change nothing should pass without regenerating. diff --git a/tests/golden/autoland/checks_fail.checkpoint.json b/tests/golden/autoland/checks_fail.checkpoint.json new file mode 100644 index 0000000..6d694b5 --- /dev/null +++ b/tests/golden/autoland/checks_fail.checkpoint.json @@ -0,0 +1,45 @@ +{ + "version": 1, + "branch": "feat", + "base": "main", + "plan_file": null, + "current_step": 0, + "last_landed_sha": "", + "abort_reason": "PR #101 checks failed after retries", + "stack": [ + { + "pr_url": "https://github.com/o/r/pull/101", + "pr_number": 101, + "branch": "TestBot/stack/101", + "title": "Add the widget model", + "review_decision": "APPROVED", + "state": "failed", + "check_retries": 1, + "queue_retries": 0, + "error_message": "Checks failed after 1 retries: ci/test" + }, + { + "pr_url": "https://github.com/o/r/pull/102", + "pr_number": 102, + "branch": "TestBot/stack/102", + "title": "Expose widgets over the API", + "review_decision": "APPROVED", + "state": "pending", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + } + ], + "plan": [ + { + "type": "land", + "entry_index": 0, + "pr_number": 101 + }, + { + "type": "land", + "entry_index": 1, + "pr_number": 102 + } + ] +} diff --git a/tests/golden/autoland/checks_fail.out.txt b/tests/golden/autoland/checks_fail.out.txt new file mode 100644 index 0000000..8e039b8 --- /dev/null +++ b/tests/golden/autoland/checks_fail.out.txt @@ -0,0 +1,33 @@ + +Discovering stack... + + +Autoland plan · 2 steps · 0 done · 2 remaining + +→ ⬜ 1. Pending | Land | #101 - Add the widget model + ⬜ 2. Pending | Land | #102 - Expose widgets over the API +State file: /home/.stack-pr/autoland/feat.json + + +============================================================ +Step 1/2: Landing PR #101 — Add the widget model +============================================================ +Rerunning failed checks (attempt 1/1): ci/test +$ gh run rerun 9101 --failed +PR #101: Failed: ci/test — polling in 120s +PR #101: Checks in progress... — polling in 120s + + + +Autoland plan · 2 steps · 0 done · 2 remaining + +→ ❌ 1. Failed | Land | #101 - Add the widget model + ↳ Checks failed after 1 retries: ci/test · retries CI 1 / MQ 0 + ⬜ 2. Pending | Land | #102 - Expose widgets over the API + +ABORTED: PR #101 checks failed after retries + +Landing failed: PR #101 checks failed after retries + +State saved to /home/.stack-pr/autoland/feat.json — resume with --resume + diff --git a/tests/golden/autoland/checks_fail.plan.txt b/tests/golden/autoland/checks_fail.plan.txt new file mode 100644 index 0000000..da890ac --- /dev/null +++ b/tests/golden/autoland/checks_fail.plan.txt @@ -0,0 +1,13 @@ +# Autoland plan — edit steps below. +# l [] = land that PR (a number, or its URL); a bare 'l' +# lands the next PR in the stack instead +# w = wait for a workflow to complete +# c [] = pause for manual confirmation; the optional +# condition names what to verify before proceeding +# (e.g. 'c QA sign-off complete') +# +# Lines starting with # are comments and are ignored. +# Blank lines are ignored. +# +l 101 # Add the widget model +l 102 # Expose widgets over the API diff --git a/tests/golden/autoland/checks_fail.status.json b/tests/golden/autoland/checks_fail.status.json new file mode 100644 index 0000000..cc12d7e --- /dev/null +++ b/tests/golden/autoland/checks_fail.status.json @@ -0,0 +1,42 @@ +{ + "status": "stopped", + "branch": "feat", + "base": "main", + "state_file": "/home/.stack-pr/autoland/feat.json", + "state_file_exists": true, + "lock_file": "/home/.stack-pr/autoland/feat.json.lock", + "pid": null, + "last_saved": "", + "abort_reason": "PR #101 checks failed after retries", + "resume_command": "stack-pr autoland --resume", + "plan": { + "total": 2, + "done": 0, + "remaining": 2, + "steps": [ + { + "number": 1, + "type": "land", + "outcome": "failed", + "status": "Failed", + "is_next": true, + "pr_number": 101, + "pr_url": "https://github.com/o/r/pull/101", + "title": "Add the widget model", + "detail": "Checks failed after 1 retries: ci/test \u00b7 retries CI 1 / MQ 0" + }, + { + "number": 2, + "type": "land", + "outcome": "pending", + "status": "Pending", + "is_next": false, + "pr_number": 102, + "pr_url": "https://github.com/o/r/pull/102", + "title": "Expose widgets over the API", + "detail": null + } + ] + }, + "other_runs": [] +} diff --git a/tests/golden/autoland/checks_fail.status.txt b/tests/golden/autoland/checks_fail.status.txt new file mode 100644 index 0000000..4ceb278 --- /dev/null +++ b/tests/golden/autoland/checks_fail.status.txt @@ -0,0 +1,18 @@ +Autoland status + + Status: Stopped — not running, can be resumed + Branch: feat + Base: main + State file: /home/.stack-pr/autoland/feat.json + Last saved: + + +Autoland plan · 2 steps · 0 done · 2 remaining + +→ ❌ 1. Failed | Land | #101 - Add the widget model + ↳ Checks failed after 1 retries: ci/test · retries CI 1 / MQ 0 + ⬜ 2. Pending | Land | #102 - Expose widgets over the API + +ABORTED: PR #101 checks failed after retries + +Resume with: stack-pr autoland --resume diff --git a/tests/golden/autoland/confirm_eof.checkpoint.json b/tests/golden/autoland/confirm_eof.checkpoint.json new file mode 100644 index 0000000..22daffe --- /dev/null +++ b/tests/golden/autoland/confirm_eof.checkpoint.json @@ -0,0 +1,50 @@ +{ + "version": 1, + "branch": "feat", + "base": "main", + "plan_file": "/plan.autoland-plan", + "current_step": 1, + "last_landed_sha": "78e4ec1063c72debbb00f2ff966f72377789cdae", + "abort_reason": "Confirm step received EOF \u2014 cannot confirm in non-interactive mode", + "stack": [ + { + "pr_url": "https://github.com/o/r/pull/101", + "pr_number": 101, + "branch": "TestBot/stack/101", + "title": "Add the widget model", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + }, + { + "pr_url": "https://github.com/o/r/pull/102", + "pr_number": 102, + "branch": "TestBot/stack/102", + "title": "Expose widgets over the API", + "review_decision": "APPROVED", + "state": "pending", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + } + ], + "plan": [ + { + "type": "land", + "entry_index": 0, + "pr_number": 101 + }, + { + "type": "confirm", + "condition": "QA sign-off complete", + "confirmed": false + }, + { + "type": "land", + "entry_index": 1, + "pr_number": 102 + } + ] +} diff --git a/tests/golden/autoland/confirm_eof.out.txt b/tests/golden/autoland/confirm_eof.out.txt new file mode 100644 index 0000000..9e986fa --- /dev/null +++ b/tests/golden/autoland/confirm_eof.out.txt @@ -0,0 +1,52 @@ + +Discovering stack... + + +Autoland plan · 3 steps · 0 done · 3 remaining + +→ ⬜ 1. Pending | Land | #101 - Add the widget model + ⬜ 2. Pending | Confirm | QA sign-off complete + ⬜ 3. Pending | Land | #102 - Expose widgets over the API +State file: /home/.stack-pr/autoland/feat.json + + +============================================================ +Step 1/3: Landing PR #101 — Add the widget model +============================================================ +PR #101 is mergeable (mergeStateStatus=CLEAN) + +Adding PR #101 to merge queue +$ gh pr merge 101 --squash +PR #101: in merge queue (0m) — polling in 120s + +PR #101 merged! + +Rebasing stack on origin/main... +$ git fetch origin main +$ git rebase origin/main +Re-submitting stack... + +============================================================ +Step 2/3: Manual confirmation required +============================================================ + +Confirm "QA sign-off complete" is complete — ready to proceed? + +Next steps: + 1. Land PR #102: Expose widgets over the API +Type y/Y then Enter to continue (Ctrl+C to abort): + + + +Autoland plan · 3 steps · 1 done · 2 remaining + + ✅ 1. Merged | Land | #101 - Add the widget model +→ ⏳ 2. Awaiting confirmation | Confirm | QA sign-off complete + ⬜ 3. Pending | Land | #102 - Expose widgets over the API + +ABORTED: Confirm step received EOF — cannot confirm in non-interactive mode + +Landing failed: Confirm step received EOF — cannot confirm in non-interactive mode + +State saved to /home/.stack-pr/autoland/feat.json — resume with --resume + diff --git a/tests/golden/autoland/confirm_eof.plan.txt b/tests/golden/autoland/confirm_eof.plan.txt new file mode 100644 index 0000000..460ebb2 --- /dev/null +++ b/tests/golden/autoland/confirm_eof.plan.txt @@ -0,0 +1,14 @@ +# Autoland plan — edit steps below. +# l [] = land that PR (a number, or its URL); a bare 'l' +# lands the next PR in the stack instead +# w = wait for a workflow to complete +# c [] = pause for manual confirmation; the optional +# condition names what to verify before proceeding +# (e.g. 'c QA sign-off complete') +# +# Lines starting with # are comments and are ignored. +# Blank lines are ignored. +# +l 101 # Add the widget model +c QA sign-off complete +l 102 # Expose widgets over the API diff --git a/tests/golden/autoland/confirm_eof.status.json b/tests/golden/autoland/confirm_eof.status.json new file mode 100644 index 0000000..109725c --- /dev/null +++ b/tests/golden/autoland/confirm_eof.status.json @@ -0,0 +1,51 @@ +{ + "status": "stopped", + "branch": "feat", + "base": "main", + "state_file": "/home/.stack-pr/autoland/feat.json", + "state_file_exists": true, + "lock_file": "/home/.stack-pr/autoland/feat.json.lock", + "pid": null, + "last_saved": "", + "abort_reason": "Confirm step received EOF \u2014 cannot confirm in non-interactive mode", + "resume_command": "stack-pr autoland --resume", + "plan": { + "total": 3, + "done": 1, + "remaining": 2, + "steps": [ + { + "number": 1, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 101, + "pr_url": "https://github.com/o/r/pull/101", + "title": "Add the widget model", + "detail": null + }, + { + "number": 2, + "type": "confirm", + "outcome": "active", + "status": "Awaiting confirmation", + "is_next": true, + "condition": "QA sign-off complete", + "detail": null + }, + { + "number": 3, + "type": "land", + "outcome": "pending", + "status": "Pending", + "is_next": false, + "pr_number": 102, + "pr_url": "https://github.com/o/r/pull/102", + "title": "Expose widgets over the API", + "detail": null + } + ] + }, + "other_runs": [] +} diff --git a/tests/golden/autoland/confirm_eof.status.txt b/tests/golden/autoland/confirm_eof.status.txt new file mode 100644 index 0000000..5a84e1c --- /dev/null +++ b/tests/golden/autoland/confirm_eof.status.txt @@ -0,0 +1,18 @@ +Autoland status + + Status: Stopped — not running, can be resumed + Branch: feat + Base: main + State file: /home/.stack-pr/autoland/feat.json + Last saved: + + +Autoland plan · 3 steps · 1 done · 2 remaining + + ✅ 1. Merged | Land | #101 - Add the widget model +→ ⏳ 2. Awaiting confirmation | Confirm | QA sign-off complete + ⬜ 3. Pending | Land | #102 - Expose widgets over the API + +ABORTED: Confirm step received EOF — cannot confirm in non-interactive mode + +Resume with: stack-pr autoland --resume diff --git a/tests/golden/autoland/landed_prefix.checkpoint.json b/tests/golden/autoland/landed_prefix.checkpoint.json new file mode 100644 index 0000000..93c027d --- /dev/null +++ b/tests/golden/autoland/landed_prefix.checkpoint.json @@ -0,0 +1,56 @@ +{ + "version": 1, + "branch": "feat", + "base": "main", + "plan_file": "/plan.autoland-plan", + "current_step": 6, + "last_landed_sha": "1a935560d359b44bfc714d90b02f064b6167c8be", + "abort_reason": "", + "stack": [ + { + "pr_url": "https://github.com/o/r/pull/103", + "pr_number": 103, + "branch": "TestBot/stack/103", + "title": "Show widgets in the dashboard", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + } + ], + "plan": [ + { + "type": "land", + "entry_index": -1, + "pr_number": 101 + }, + { + "type": "workflow", + "workflow": "deploy.yaml", + "state": "skipped", + "error_message": "" + }, + { + "type": "confirm", + "condition": "QA sign-off complete", + "confirmed": true + }, + { + "type": "land", + "entry_index": -1, + "pr_number": 102 + }, + { + "type": "workflow", + "workflow": "deploy.yaml", + "state": "succeeded", + "error_message": "" + }, + { + "type": "land", + "entry_index": 0, + "pr_number": 103 + } + ] +} diff --git a/tests/golden/autoland/landed_prefix.out.txt b/tests/golden/autoland/landed_prefix.out.txt new file mode 100644 index 0000000..a68607b --- /dev/null +++ b/tests/golden/autoland/landed_prefix.out.txt @@ -0,0 +1,53 @@ + +Discovering stack... + + +Autoland plan · 6 steps · 4 done · 2 remaining + + ✅ 1. Landed | Land | #101 + ✅ 2. Assumed completed | Workflow | deploy.yaml + ✅ 3. Assumed completed | Confirm | QA sign-off complete + ✅ 4. Landed | Land | #102 +→ ⬜ 5. Pending | Workflow | deploy.yaml + ⬜ 6. Pending | Land | #103 - Show widgets in the dashboard +State file: /home/.stack-pr/autoland/feat.json + + +PR #101 already landed, skipping + +PR #102 already landed, skipping + +============================================================ +Step 5/6: Workflow checkpoint — deploy.yaml +============================================================ + +Waiting for workflow: deploy.yaml +Target SHA: 9cc0a8c62f2d +Workflow deploy.yaml: waiting (0m) — polling in 120s + +Workflow deploy.yaml completed with SHA 9cc0a8c62f2d + +============================================================ +Step 6/6: Landing PR #103 — Show widgets in the dashboard +============================================================ +PR #103 is mergeable (mergeStateStatus=CLEAN) + +Adding PR #103 to merge queue +$ gh pr merge 103 --squash +PR #103: in merge queue (0m) — polling in 120s + +PR #103 merged! + + + +Autoland plan · 6 steps · 6 done · 0 remaining + + ✅ 1. Landed | Land | #101 + ✅ 2. Assumed completed | Workflow | deploy.yaml + ✅ 3. Assumed completed | Confirm | QA sign-off complete + ✅ 4. Landed | Land | #102 + ✅ 5. Workflow complete | Workflow | deploy.yaml + ✅ 6. Merged | Land | #103 - Show widgets in the dashboard + +All PRs landed successfully! + diff --git a/tests/golden/autoland/landed_prefix.plan.txt b/tests/golden/autoland/landed_prefix.plan.txt new file mode 100644 index 0000000..f22249b --- /dev/null +++ b/tests/golden/autoland/landed_prefix.plan.txt @@ -0,0 +1,17 @@ +# Autoland plan — edit steps below. +# l [] = land that PR (a number, or its URL); a bare 'l' +# lands the next PR in the stack instead +# w = wait for a workflow to complete +# c [] = pause for manual confirmation; the optional +# condition names what to verify before proceeding +# (e.g. 'c QA sign-off complete') +# +# Lines starting with # are comments and are ignored. +# Blank lines are ignored. +# +l 101 # already landed +w deploy.yaml +c QA sign-off complete +l 102 # already landed +w deploy.yaml +l 103 # Show widgets in the dashboard diff --git a/tests/golden/autoland/landed_prefix.status.json b/tests/golden/autoland/landed_prefix.status.json new file mode 100644 index 0000000..7a24661 --- /dev/null +++ b/tests/golden/autoland/landed_prefix.status.json @@ -0,0 +1,80 @@ +{ + "status": "stopped", + "branch": "feat", + "base": "main", + "state_file": "/home/.stack-pr/autoland/feat.json", + "state_file_exists": true, + "lock_file": "/home/.stack-pr/autoland/feat.json.lock", + "pid": null, + "last_saved": "", + "abort_reason": null, + "resume_command": "stack-pr autoland --resume", + "plan": { + "total": 6, + "done": 6, + "remaining": 0, + "steps": [ + { + "number": 1, + "type": "land", + "outcome": "done", + "status": "Landed", + "is_next": false, + "pr_number": 101, + "pr_url": null, + "title": null, + "detail": null + }, + { + "number": 2, + "type": "workflow", + "outcome": "done", + "status": "Assumed completed", + "is_next": false, + "workflow": "deploy.yaml", + "detail": null + }, + { + "number": 3, + "type": "confirm", + "outcome": "done", + "status": "Assumed completed", + "is_next": false, + "condition": "QA sign-off complete", + "detail": null + }, + { + "number": 4, + "type": "land", + "outcome": "done", + "status": "Landed", + "is_next": false, + "pr_number": 102, + "pr_url": null, + "title": null, + "detail": null + }, + { + "number": 5, + "type": "workflow", + "outcome": "done", + "status": "Workflow complete", + "is_next": false, + "workflow": "deploy.yaml", + "detail": null + }, + { + "number": 6, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 103, + "pr_url": "https://github.com/o/r/pull/103", + "title": "Show widgets in the dashboard", + "detail": null + } + ] + }, + "other_runs": [] +} diff --git a/tests/golden/autoland/landed_prefix.status.txt b/tests/golden/autoland/landed_prefix.status.txt new file mode 100644 index 0000000..9008205 --- /dev/null +++ b/tests/golden/autoland/landed_prefix.status.txt @@ -0,0 +1,19 @@ +Autoland status + + Status: Stopped — not running, can be resumed + Branch: feat + Base: main + State file: /home/.stack-pr/autoland/feat.json + Last saved: + + +Autoland plan · 6 steps · 6 done · 0 remaining + + ✅ 1. Landed | Land | #101 + ✅ 2. Assumed completed | Workflow | deploy.yaml + ✅ 3. Assumed completed | Confirm | QA sign-off complete + ✅ 4. Landed | Land | #102 + ✅ 5. Workflow complete | Workflow | deploy.yaml + ✅ 6. Merged | Land | #103 - Show widgets in the dashboard + +Resume with: stack-pr autoland --resume diff --git a/tests/golden/autoland/merge_as_stack.checkpoint.json b/tests/golden/autoland/merge_as_stack.checkpoint.json new file mode 100644 index 0000000..62c408b --- /dev/null +++ b/tests/golden/autoland/merge_as_stack.checkpoint.json @@ -0,0 +1,72 @@ +{ + "version": 1, + "branch": "feat", + "base": "main", + "plan_file": "/plan.autoland-plan", + "current_step": 5, + "last_landed_sha": "1a935560d359b44bfc714d90b02f064b6167c8be", + "abort_reason": "", + "stack": [ + { + "pr_url": "https://github.com/o/r/pull/101", + "pr_number": 101, + "branch": "TestBot/stack/101", + "title": "Add the widget model", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + }, + { + "pr_url": "https://github.com/o/r/pull/102", + "pr_number": 102, + "branch": "TestBot/stack/102", + "title": "Expose widgets over the API", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + }, + { + "pr_url": "https://github.com/o/r/pull/103", + "pr_number": 103, + "branch": "TestBot/stack/103", + "title": "Show widgets in the dashboard", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + } + ], + "plan": [ + { + "type": "land", + "entry_index": 0, + "pr_number": 101 + }, + { + "type": "land", + "entry_index": 1, + "pr_number": 102 + }, + { + "type": "workflow", + "workflow": "deploy.yaml", + "state": "succeeded", + "error_message": "" + }, + { + "type": "confirm", + "condition": "", + "confirmed": true + }, + { + "type": "land", + "entry_index": 2, + "pr_number": 103 + } + ] +} diff --git a/tests/golden/autoland/merge_as_stack.out.txt b/tests/golden/autoland/merge_as_stack.out.txt new file mode 100644 index 0000000..2b607a3 --- /dev/null +++ b/tests/golden/autoland/merge_as_stack.out.txt @@ -0,0 +1,78 @@ + +Discovering stack... + + +Autoland plan · 5 steps · 0 done · 5 remaining + +→ ⬜ 1. Pending | Land | #101 - Add the widget model + ⬜ 2. Pending | Land | #102 - Expose widgets over the API + ⬜ 3. Pending | Workflow | deploy.yaml + ⬜ 4. Pending | Confirm | (no condition) + ⬜ 5. Pending | Land | #103 - Show widgets in the dashboard +Steps 1-2 (#101, #102) will merge together as a GitHub stack. +State file: /home/.stack-pr/autoland/feat.json + + +============================================================ +Landing 2 PRs as a GitHub stack: #101, #102 +============================================================ +PR #101 is mergeable (mergeStateStatus=CLEAN) + +Merging PRs #101-#102 as a stack +Stack of 2 PRs up to #102: enqueued (0m) — polling in 120s + +PRs #101, #102 merged! + +Rebasing stack on origin/main... +$ git fetch origin main +$ git rebase origin/main +Re-submitting stack... + +PR #101 already merged, skipping + +PR #102 already merged, skipping + +============================================================ +Step 3/5: Workflow checkpoint — deploy.yaml +============================================================ + +Waiting for workflow: deploy.yaml +Target SHA: 9cc0a8c62f2d +Workflow deploy.yaml: waiting (0m) — polling in 120s + +Workflow deploy.yaml completed with SHA 9cc0a8c62f2d + +============================================================ +Step 4/5: Manual confirmation required +============================================================ + +Ready to proceed? + +Next steps: + 1. Land PR #103: Show widgets in the dashboard +Type y/Y then Enter to continue (Ctrl+C to abort): y +Confirmed + +============================================================ +Step 5/5: Landing PR #103 — Show widgets in the dashboard +============================================================ +PR #103 is mergeable (mergeStateStatus=CLEAN) + +Adding PR #103 to merge queue +$ gh pr merge 103 --squash +PR #103: in merge queue (0m) — polling in 120s + +PR #103 merged! + + + +Autoland plan · 5 steps · 5 done · 0 remaining + + ✅ 1. Merged | Land | #101 - Add the widget model + ✅ 2. Merged | Land | #102 - Expose widgets over the API + ✅ 3. Workflow complete | Workflow | deploy.yaml + ✅ 4. Confirmed | Confirm | (no condition) + ✅ 5. Merged | Land | #103 - Show widgets in the dashboard + +All PRs landed successfully! + diff --git a/tests/golden/autoland/merge_as_stack.plan.txt b/tests/golden/autoland/merge_as_stack.plan.txt new file mode 100644 index 0000000..070b643 --- /dev/null +++ b/tests/golden/autoland/merge_as_stack.plan.txt @@ -0,0 +1,16 @@ +# Autoland plan — edit steps below. +# l [] = land that PR (a number, or its URL); a bare 'l' +# lands the next PR in the stack instead +# w = wait for a workflow to complete +# c [] = pause for manual confirmation; the optional +# condition names what to verify before proceeding +# (e.g. 'c QA sign-off complete') +# +# Lines starting with # are comments and are ignored. +# Blank lines are ignored. +# +l 101 # Add the widget model +l 102 # Expose widgets over the API +w deploy.yaml +c +l 103 # Show widgets in the dashboard diff --git a/tests/golden/autoland/merge_as_stack.status.json b/tests/golden/autoland/merge_as_stack.status.json new file mode 100644 index 0000000..47a06be --- /dev/null +++ b/tests/golden/autoland/merge_as_stack.status.json @@ -0,0 +1,71 @@ +{ + "status": "stopped", + "branch": "feat", + "base": "main", + "state_file": "/home/.stack-pr/autoland/feat.json", + "state_file_exists": true, + "lock_file": "/home/.stack-pr/autoland/feat.json.lock", + "pid": null, + "last_saved": "", + "abort_reason": null, + "resume_command": "stack-pr autoland --resume", + "plan": { + "total": 5, + "done": 5, + "remaining": 0, + "steps": [ + { + "number": 1, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 101, + "pr_url": "https://github.com/o/r/pull/101", + "title": "Add the widget model", + "detail": null + }, + { + "number": 2, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 102, + "pr_url": "https://github.com/o/r/pull/102", + "title": "Expose widgets over the API", + "detail": null + }, + { + "number": 3, + "type": "workflow", + "outcome": "done", + "status": "Workflow complete", + "is_next": false, + "workflow": "deploy.yaml", + "detail": null + }, + { + "number": 4, + "type": "confirm", + "outcome": "done", + "status": "Confirmed", + "is_next": false, + "condition": "", + "detail": null + }, + { + "number": 5, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 103, + "pr_url": "https://github.com/o/r/pull/103", + "title": "Show widgets in the dashboard", + "detail": null + } + ] + }, + "other_runs": [] +} diff --git a/tests/golden/autoland/merge_as_stack.status.txt b/tests/golden/autoland/merge_as_stack.status.txt new file mode 100644 index 0000000..f0efc2d --- /dev/null +++ b/tests/golden/autoland/merge_as_stack.status.txt @@ -0,0 +1,18 @@ +Autoland status + + Status: Stopped — not running, can be resumed + Branch: feat + Base: main + State file: /home/.stack-pr/autoland/feat.json + Last saved: + + +Autoland plan · 5 steps · 5 done · 0 remaining + + ✅ 1. Merged | Land | #101 - Add the widget model + ✅ 2. Merged | Land | #102 - Expose widgets over the API + ✅ 3. Workflow complete | Workflow | deploy.yaml + ✅ 4. Confirmed | Confirm | (no condition) + ✅ 5. Merged | Land | #103 - Show widgets in the dashboard + +Resume with: stack-pr autoland --resume diff --git a/tests/golden/autoland/one_at_a_time.checkpoint.json b/tests/golden/autoland/one_at_a_time.checkpoint.json new file mode 100644 index 0000000..a49e68b --- /dev/null +++ b/tests/golden/autoland/one_at_a_time.checkpoint.json @@ -0,0 +1,56 @@ +{ + "version": 1, + "branch": "feat", + "base": "main", + "plan_file": "/plan.autoland-plan", + "current_step": 4, + "last_landed_sha": "9cc0a8c62f2d99c85284a96fe68191bd3843f14b", + "abort_reason": "", + "stack": [ + { + "pr_url": "https://github.com/o/r/pull/101", + "pr_number": 101, + "branch": "TestBot/stack/101", + "title": "Add the widget model", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + }, + { + "pr_url": "https://github.com/o/r/pull/102", + "pr_number": 102, + "branch": "TestBot/stack/102", + "title": "Expose widgets over the API", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + } + ], + "plan": [ + { + "type": "land", + "entry_index": 0, + "pr_number": 101 + }, + { + "type": "workflow", + "workflow": "deploy.yaml", + "state": "succeeded", + "error_message": "" + }, + { + "type": "confirm", + "condition": "QA sign-off complete", + "confirmed": true + }, + { + "type": "land", + "entry_index": 1, + "pr_number": 102 + } + ] +} diff --git a/tests/golden/autoland/one_at_a_time.out.txt b/tests/golden/autoland/one_at_a_time.out.txt new file mode 100644 index 0000000..477a436 --- /dev/null +++ b/tests/golden/autoland/one_at_a_time.out.txt @@ -0,0 +1,78 @@ + +Discovering stack... + + +Autoland plan · 4 steps · 0 done · 4 remaining + +→ ⬜ 1. Pending | Land | #101 - Add the widget model + ⬜ 2. Pending | Workflow | deploy.yaml + ⬜ 3. Pending | Confirm | QA sign-off complete + ⬜ 4. Pending | Land | #102 - Expose widgets over the API +State file: /home/.stack-pr/autoland/feat.json + + +============================================================ +Step 1/4: Landing PR #101 — Add the widget model +============================================================ +PR #101: Checks in progress... — polling in 120s +PR #101 is mergeable (mergeStateStatus=CLEAN) + +Adding PR #101 to merge queue +$ gh pr merge 101 --squash +PR #101: in merge queue (0m) — polling in 120s + +PR #101 merged! + +Rebasing stack on origin/main... +$ git fetch origin main +$ git rebase origin/main +Re-submitting stack... + +============================================================ +Step 2/4: Workflow checkpoint — deploy.yaml +============================================================ + +Waiting for workflow: deploy.yaml +Target SHA: 78e4ec1063c7 +Workflow deploy.yaml: waiting (0m) — polling in 120s + +Workflow deploy.yaml completed with SHA 78e4ec1063c7 + +============================================================ +Step 3/4: Manual confirmation required +============================================================ + +Confirm "QA sign-off complete" is complete — ready to proceed? + +Next steps: + 1. Land PR #102: Expose widgets over the API +Type y/Y then Enter to continue (Ctrl+C to abort): n +Type 'y' or 'Y' to confirm. +Type y/Y then Enter to continue (Ctrl+C to abort): y +Confirmed + +============================================================ +Step 4/4: Landing PR #102 — Expose widgets over the API +============================================================ +PR #102 is not yet approved (REVIEW_REQUIRED). Waiting... +PR #102: waiting for approval — polling in 120s +PR #102 is now approved +PR #102 is mergeable (mergeStateStatus=CLEAN) + +Adding PR #102 to merge queue +$ gh pr merge 102 --squash +PR #102: in merge queue (0m) — polling in 120s + +PR #102 merged! + + + +Autoland plan · 4 steps · 4 done · 0 remaining + + ✅ 1. Merged | Land | #101 - Add the widget model + ✅ 2. Workflow complete | Workflow | deploy.yaml + ✅ 3. Confirmed | Confirm | QA sign-off complete + ✅ 4. Merged | Land | #102 - Expose widgets over the API + +All PRs landed successfully! + diff --git a/tests/golden/autoland/one_at_a_time.plan.txt b/tests/golden/autoland/one_at_a_time.plan.txt new file mode 100644 index 0000000..f77bb05 --- /dev/null +++ b/tests/golden/autoland/one_at_a_time.plan.txt @@ -0,0 +1,15 @@ +# Autoland plan — edit steps below. +# l [] = land that PR (a number, or its URL); a bare 'l' +# lands the next PR in the stack instead +# w = wait for a workflow to complete +# c [] = pause for manual confirmation; the optional +# condition names what to verify before proceeding +# (e.g. 'c QA sign-off complete') +# +# Lines starting with # are comments and are ignored. +# Blank lines are ignored. +# +l 101 # Add the widget model +w deploy.yaml +c QA sign-off complete +l 102 # Expose widgets over the API diff --git a/tests/golden/autoland/one_at_a_time.status.json b/tests/golden/autoland/one_at_a_time.status.json new file mode 100644 index 0000000..cd3ca1a --- /dev/null +++ b/tests/golden/autoland/one_at_a_time.status.json @@ -0,0 +1,60 @@ +{ + "status": "stopped", + "branch": "feat", + "base": "main", + "state_file": "/home/.stack-pr/autoland/feat.json", + "state_file_exists": true, + "lock_file": "/home/.stack-pr/autoland/feat.json.lock", + "pid": null, + "last_saved": "", + "abort_reason": null, + "resume_command": "stack-pr autoland --resume", + "plan": { + "total": 4, + "done": 4, + "remaining": 0, + "steps": [ + { + "number": 1, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 101, + "pr_url": "https://github.com/o/r/pull/101", + "title": "Add the widget model", + "detail": null + }, + { + "number": 2, + "type": "workflow", + "outcome": "done", + "status": "Workflow complete", + "is_next": false, + "workflow": "deploy.yaml", + "detail": null + }, + { + "number": 3, + "type": "confirm", + "outcome": "done", + "status": "Confirmed", + "is_next": false, + "condition": "QA sign-off complete", + "detail": null + }, + { + "number": 4, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 102, + "pr_url": "https://github.com/o/r/pull/102", + "title": "Expose widgets over the API", + "detail": null + } + ] + }, + "other_runs": [] +} diff --git a/tests/golden/autoland/one_at_a_time.status.txt b/tests/golden/autoland/one_at_a_time.status.txt new file mode 100644 index 0000000..47ca38f --- /dev/null +++ b/tests/golden/autoland/one_at_a_time.status.txt @@ -0,0 +1,17 @@ +Autoland status + + Status: Stopped — not running, can be resumed + Branch: feat + Base: main + State file: /home/.stack-pr/autoland/feat.json + Last saved: + + +Autoland plan · 4 steps · 4 done · 0 remaining + + ✅ 1. Merged | Land | #101 - Add the widget model + ✅ 2. Workflow complete | Workflow | deploy.yaml + ✅ 3. Confirmed | Confirm | QA sign-off complete + ✅ 4. Merged | Land | #102 - Expose widgets over the API + +Resume with: stack-pr autoland --resume diff --git a/tests/golden/autoland/replan_carry_over.checkpoint.json b/tests/golden/autoland/replan_carry_over.checkpoint.json new file mode 100644 index 0000000..978f751 --- /dev/null +++ b/tests/golden/autoland/replan_carry_over.checkpoint.json @@ -0,0 +1,66 @@ +{ + "version": 1, + "branch": "feat", + "base": "main", + "plan_file": "/plan.autoland-plan", + "current_step": 8, + "last_landed_sha": "1a935560d359b44bfc714d90b02f064b6167c8be", + "abort_reason": "", + "stack": [ + { + "pr_url": "https://github.com/o/r/pull/103", + "pr_number": 103, + "branch": "TestBot/stack/103", + "title": "Show widgets in the dashboard", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + } + ], + "plan": [ + { + "type": "land", + "entry_index": -1, + "pr_number": 101 + }, + { + "type": "workflow", + "workflow": "deploy.yaml", + "state": "skipped", + "error_message": "" + }, + { + "type": "confirm", + "condition": "QA sign-off complete", + "confirmed": true + }, + { + "type": "land", + "entry_index": -1, + "pr_number": 102 + }, + { + "type": "workflow", + "workflow": "deploy.yaml", + "state": "succeeded", + "error_message": "" + }, + { + "type": "confirm", + "condition": "metrics look healthy", + "confirmed": true + }, + { + "type": "confirm", + "condition": "docs updated", + "confirmed": true + }, + { + "type": "land", + "entry_index": 0, + "pr_number": 103 + } + ] +} diff --git a/tests/golden/autoland/replan_carry_over.out.txt b/tests/golden/autoland/replan_carry_over.out.txt new file mode 100644 index 0000000..b9694d7 --- /dev/null +++ b/tests/golden/autoland/replan_carry_over.out.txt @@ -0,0 +1,65 @@ +Replanning from checkpoint: /home/.stack-pr/autoland/feat.json + +Rediscovering stack... + +Plan: /plan.autoland-plan + +Autoland plan · 8 steps · 6 done · 2 remaining + + ✅ 1. Landed | Land | #101 + ✅ 2. Assumed completed | Workflow | deploy.yaml + ✅ 3. Assumed completed | Confirm | QA sign-off complete + ✅ 4. Landed | Land | #102 +→ ✅ 5. Workflow complete | Workflow | deploy.yaml + ✅ 6. Confirmed | Confirm | metrics look healthy + ⬜ 7. Pending | Confirm | docs updated + ⬜ 8. Pending | Land | #103 - Show widgets in the dashboard + +Done in the previous run, but not carried over (changed, removed, or now after different PRs): + - Manual confirmation: on-call agrees (after #101, #102) + +Continue with this plan? Type y/Y to confirm (anything else aborts): y +State file: /home/.stack-pr/autoland/feat.json + + +PR #101 already landed, skipping + +PR #102 already landed, skipping + +============================================================ +Step 7/8: Manual confirmation required +============================================================ + +Confirm "docs updated" is complete — ready to proceed? + +Next steps: + 1. Land PR #103: Show widgets in the dashboard +Type y/Y then Enter to continue (Ctrl+C to abort): y +Confirmed + +============================================================ +Step 8/8: Landing PR #103 — Show widgets in the dashboard +============================================================ +PR #103 is mergeable (mergeStateStatus=CLEAN) + +Adding PR #103 to merge queue +$ gh pr merge 103 --squash +PR #103: in merge queue (0m) — polling in 120s + +PR #103 merged! + + + +Autoland plan · 8 steps · 8 done · 0 remaining + + ✅ 1. Landed | Land | #101 + ✅ 2. Assumed completed | Workflow | deploy.yaml + ✅ 3. Assumed completed | Confirm | QA sign-off complete + ✅ 4. Landed | Land | #102 + ✅ 5. Workflow complete | Workflow | deploy.yaml + ✅ 6. Confirmed | Confirm | metrics look healthy + ✅ 7. Confirmed | Confirm | docs updated + ✅ 8. Merged | Land | #103 - Show widgets in the dashboard + +All PRs landed successfully! + diff --git a/tests/golden/autoland/replan_carry_over.plan.txt b/tests/golden/autoland/replan_carry_over.plan.txt new file mode 100644 index 0000000..96d22eb --- /dev/null +++ b/tests/golden/autoland/replan_carry_over.plan.txt @@ -0,0 +1,19 @@ +# Autoland plan — edit steps below. +# l [] = land that PR (a number, or its URL); a bare 'l' +# lands the next PR in the stack instead +# w = wait for a workflow to complete +# c [] = pause for manual confirmation; the optional +# condition names what to verify before proceeding +# (e.g. 'c QA sign-off complete') +# +# Lines starting with # are comments and are ignored. +# Blank lines are ignored. +# +l 101 # already landed +w deploy.yaml +c QA sign-off complete +l 102 # already landed +w deploy.yaml +c metrics look healthy +c docs updated +l 103 # Show widgets in the dashboard diff --git a/tests/golden/autoland/replan_carry_over.status.json b/tests/golden/autoland/replan_carry_over.status.json new file mode 100644 index 0000000..5270207 --- /dev/null +++ b/tests/golden/autoland/replan_carry_over.status.json @@ -0,0 +1,98 @@ +{ + "status": "stopped", + "branch": "feat", + "base": "main", + "state_file": "/home/.stack-pr/autoland/feat.json", + "state_file_exists": true, + "lock_file": "/home/.stack-pr/autoland/feat.json.lock", + "pid": null, + "last_saved": "", + "abort_reason": null, + "resume_command": "stack-pr autoland --resume", + "plan": { + "total": 8, + "done": 8, + "remaining": 0, + "steps": [ + { + "number": 1, + "type": "land", + "outcome": "done", + "status": "Landed", + "is_next": false, + "pr_number": 101, + "pr_url": null, + "title": null, + "detail": null + }, + { + "number": 2, + "type": "workflow", + "outcome": "done", + "status": "Assumed completed", + "is_next": false, + "workflow": "deploy.yaml", + "detail": null + }, + { + "number": 3, + "type": "confirm", + "outcome": "done", + "status": "Assumed completed", + "is_next": false, + "condition": "QA sign-off complete", + "detail": null + }, + { + "number": 4, + "type": "land", + "outcome": "done", + "status": "Landed", + "is_next": false, + "pr_number": 102, + "pr_url": null, + "title": null, + "detail": null + }, + { + "number": 5, + "type": "workflow", + "outcome": "done", + "status": "Workflow complete", + "is_next": false, + "workflow": "deploy.yaml", + "detail": null + }, + { + "number": 6, + "type": "confirm", + "outcome": "done", + "status": "Confirmed", + "is_next": false, + "condition": "metrics look healthy", + "detail": null + }, + { + "number": 7, + "type": "confirm", + "outcome": "done", + "status": "Confirmed", + "is_next": false, + "condition": "docs updated", + "detail": null + }, + { + "number": 8, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 103, + "pr_url": "https://github.com/o/r/pull/103", + "title": "Show widgets in the dashboard", + "detail": null + } + ] + }, + "other_runs": [] +} diff --git a/tests/golden/autoland/replan_carry_over.status.txt b/tests/golden/autoland/replan_carry_over.status.txt new file mode 100644 index 0000000..49df746 --- /dev/null +++ b/tests/golden/autoland/replan_carry_over.status.txt @@ -0,0 +1,21 @@ +Autoland status + + Status: Stopped — not running, can be resumed + Branch: feat + Base: main + State file: /home/.stack-pr/autoland/feat.json + Last saved: + + +Autoland plan · 8 steps · 8 done · 0 remaining + + ✅ 1. Landed | Land | #101 + ✅ 2. Assumed completed | Workflow | deploy.yaml + ✅ 3. Assumed completed | Confirm | QA sign-off complete + ✅ 4. Landed | Land | #102 + ✅ 5. Workflow complete | Workflow | deploy.yaml + ✅ 6. Confirmed | Confirm | metrics look healthy + ✅ 7. Confirmed | Confirm | docs updated + ✅ 8. Merged | Land | #103 - Show widgets in the dashboard + +Resume with: stack-pr autoland --resume diff --git a/tests/golden/autoland/resume_mid_plan.checkpoint.json b/tests/golden/autoland/resume_mid_plan.checkpoint.json new file mode 100644 index 0000000..581484a --- /dev/null +++ b/tests/golden/autoland/resume_mid_plan.checkpoint.json @@ -0,0 +1,56 @@ +{ + "version": 1, + "branch": "feat", + "base": "main", + "plan_file": null, + "current_step": 6, + "last_landed_sha": "1a935560d359b44bfc714d90b02f064b6167c8be", + "abort_reason": "", + "stack": [ + { + "pr_url": "https://github.com/o/r/pull/103", + "pr_number": 103, + "branch": "TestBot/stack/103", + "title": "Show widgets in the dashboard", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + } + ], + "plan": [ + { + "type": "land", + "entry_index": -1, + "pr_number": 101 + }, + { + "type": "workflow", + "workflow": "deploy.yaml", + "state": "skipped", + "error_message": "" + }, + { + "type": "confirm", + "condition": "QA sign-off complete", + "confirmed": true + }, + { + "type": "land", + "entry_index": -1, + "pr_number": 102 + }, + { + "type": "workflow", + "workflow": "deploy.yaml", + "state": "succeeded", + "error_message": "" + }, + { + "type": "land", + "entry_index": 0, + "pr_number": 103 + } + ] +} diff --git a/tests/golden/autoland/resume_mid_plan.out.txt b/tests/golden/autoland/resume_mid_plan.out.txt new file mode 100644 index 0000000..1ebc1ed --- /dev/null +++ b/tests/golden/autoland/resume_mid_plan.out.txt @@ -0,0 +1,50 @@ +Resuming from checkpoint: /home/.stack-pr/autoland/feat.json + +Refreshing PR state from GitHub... + +Autoland plan · 6 steps · 4 done · 2 remaining + + ✅ 1. Landed | Land | #101 + ✅ 2. Assumed completed | Workflow | deploy.yaml + ✅ 3. Assumed completed | Confirm | QA sign-off complete + ✅ 4. Landed | Land | #102 +→ ❌ 5. Failed | Workflow | deploy.yaml + ↳ Workflow timed out after 3h + ⬜ 6. Pending | Land | #103 - Show widgets in the dashboard +State file: /home/.stack-pr/autoland/feat.json + + +============================================================ +Step 5/6: Workflow checkpoint — deploy.yaml +============================================================ + +Waiting for workflow: deploy.yaml +Target SHA: 9cc0a8c62f2d +Workflow deploy.yaml: waiting (0m) — polling in 120s + +Workflow deploy.yaml completed with SHA 9cc0a8c62f2d + +============================================================ +Step 6/6: Landing PR #103 — Show widgets in the dashboard +============================================================ +PR #103 is mergeable (mergeStateStatus=CLEAN) + +Adding PR #103 to merge queue +$ gh pr merge 103 --squash +PR #103: in merge queue (0m) — polling in 120s + +PR #103 merged! + + + +Autoland plan · 6 steps · 6 done · 0 remaining + + ✅ 1. Landed | Land | #101 + ✅ 2. Assumed completed | Workflow | deploy.yaml + ✅ 3. Assumed completed | Confirm | QA sign-off complete + ✅ 4. Landed | Land | #102 + ✅ 5. Workflow complete | Workflow | deploy.yaml + ✅ 6. Merged | Land | #103 - Show widgets in the dashboard + +All PRs landed successfully! + diff --git a/tests/golden/autoland/resume_mid_plan.plan.txt b/tests/golden/autoland/resume_mid_plan.plan.txt new file mode 100644 index 0000000..f22249b --- /dev/null +++ b/tests/golden/autoland/resume_mid_plan.plan.txt @@ -0,0 +1,17 @@ +# Autoland plan — edit steps below. +# l [] = land that PR (a number, or its URL); a bare 'l' +# lands the next PR in the stack instead +# w = wait for a workflow to complete +# c [] = pause for manual confirmation; the optional +# condition names what to verify before proceeding +# (e.g. 'c QA sign-off complete') +# +# Lines starting with # are comments and are ignored. +# Blank lines are ignored. +# +l 101 # already landed +w deploy.yaml +c QA sign-off complete +l 102 # already landed +w deploy.yaml +l 103 # Show widgets in the dashboard diff --git a/tests/golden/autoland/resume_mid_plan.status.json b/tests/golden/autoland/resume_mid_plan.status.json new file mode 100644 index 0000000..7a24661 --- /dev/null +++ b/tests/golden/autoland/resume_mid_plan.status.json @@ -0,0 +1,80 @@ +{ + "status": "stopped", + "branch": "feat", + "base": "main", + "state_file": "/home/.stack-pr/autoland/feat.json", + "state_file_exists": true, + "lock_file": "/home/.stack-pr/autoland/feat.json.lock", + "pid": null, + "last_saved": "", + "abort_reason": null, + "resume_command": "stack-pr autoland --resume", + "plan": { + "total": 6, + "done": 6, + "remaining": 0, + "steps": [ + { + "number": 1, + "type": "land", + "outcome": "done", + "status": "Landed", + "is_next": false, + "pr_number": 101, + "pr_url": null, + "title": null, + "detail": null + }, + { + "number": 2, + "type": "workflow", + "outcome": "done", + "status": "Assumed completed", + "is_next": false, + "workflow": "deploy.yaml", + "detail": null + }, + { + "number": 3, + "type": "confirm", + "outcome": "done", + "status": "Assumed completed", + "is_next": false, + "condition": "QA sign-off complete", + "detail": null + }, + { + "number": 4, + "type": "land", + "outcome": "done", + "status": "Landed", + "is_next": false, + "pr_number": 102, + "pr_url": null, + "title": null, + "detail": null + }, + { + "number": 5, + "type": "workflow", + "outcome": "done", + "status": "Workflow complete", + "is_next": false, + "workflow": "deploy.yaml", + "detail": null + }, + { + "number": 6, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 103, + "pr_url": "https://github.com/o/r/pull/103", + "title": "Show widgets in the dashboard", + "detail": null + } + ] + }, + "other_runs": [] +} diff --git a/tests/golden/autoland/resume_mid_plan.status.txt b/tests/golden/autoland/resume_mid_plan.status.txt new file mode 100644 index 0000000..9008205 --- /dev/null +++ b/tests/golden/autoland/resume_mid_plan.status.txt @@ -0,0 +1,19 @@ +Autoland status + + Status: Stopped — not running, can be resumed + Branch: feat + Base: main + State file: /home/.stack-pr/autoland/feat.json + Last saved: + + +Autoland plan · 6 steps · 6 done · 0 remaining + + ✅ 1. Landed | Land | #101 + ✅ 2. Assumed completed | Workflow | deploy.yaml + ✅ 3. Assumed completed | Confirm | QA sign-off complete + ✅ 4. Landed | Land | #102 + ✅ 5. Workflow complete | Workflow | deploy.yaml + ✅ 6. Merged | Land | #103 - Show widgets in the dashboard + +Resume with: stack-pr autoland --resume diff --git a/tests/golden/autoland/stack_merge_falls_through.checkpoint.json b/tests/golden/autoland/stack_merge_falls_through.checkpoint.json new file mode 100644 index 0000000..ed9efe9 --- /dev/null +++ b/tests/golden/autoland/stack_merge_falls_through.checkpoint.json @@ -0,0 +1,61 @@ +{ + "version": 1, + "branch": "feat", + "base": "main", + "plan_file": null, + "current_step": 3, + "last_landed_sha": "1a935560d359b44bfc714d90b02f064b6167c8be", + "abort_reason": "", + "stack": [ + { + "pr_url": "https://github.com/o/r/pull/101", + "pr_number": 101, + "branch": "TestBot/stack/101", + "title": "Add the widget model", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + }, + { + "pr_url": "https://github.com/o/r/pull/102", + "pr_number": 102, + "branch": "TestBot/stack/102", + "title": "Expose widgets over the API", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + }, + { + "pr_url": "https://github.com/o/r/pull/103", + "pr_number": 103, + "branch": "TestBot/stack/103", + "title": "Show widgets in the dashboard", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + } + ], + "plan": [ + { + "type": "land", + "entry_index": 0, + "pr_number": 101 + }, + { + "type": "land", + "entry_index": 1, + "pr_number": 102 + }, + { + "type": "land", + "entry_index": 2, + "pr_number": 103 + } + ] +} diff --git a/tests/golden/autoland/stack_merge_falls_through.out.txt b/tests/golden/autoland/stack_merge_falls_through.out.txt new file mode 100644 index 0000000..43bf24d --- /dev/null +++ b/tests/golden/autoland/stack_merge_falls_through.out.txt @@ -0,0 +1,65 @@ + +Discovering stack... + + +Autoland plan · 3 steps · 0 done · 3 remaining + +→ ⬜ 1. Pending | Land | #101 - Add the widget model + ⬜ 2. Pending | Land | #102 - Expose widgets over the API + ⬜ 3. Pending | Land | #103 - Show widgets in the dashboard +Steps 1-3 (#101, #102, #103) will merge together as a GitHub stack. +State file: /home/.stack-pr/autoland/feat.json + + +============================================================ +Landing 3 PRs as a GitHub stack: #101, #102, #103 +============================================================ +PR #101 is mergeable (mergeStateStatus=CLEAN) + +Merging PRs #101-#103 as a stack +Stack merge did not complete (Merge conflict in PR #102). Landing the remaining PRs one at a time. + +Rebasing stack on origin/main... +$ git fetch origin main +$ git rebase origin/main +Re-submitting stack... + +PR #101 already merged, skipping + +============================================================ +Step 2/3: Landing PR #102 — Expose widgets over the API +============================================================ +PR #102 is mergeable (mergeStateStatus=CLEAN) + +Adding PR #102 to merge queue +$ gh pr merge 102 --squash +PR #102: in merge queue (0m) — polling in 120s + +PR #102 merged! + +Rebasing stack on origin/main... +$ git fetch origin main +$ git rebase origin/main +Re-submitting stack... + +============================================================ +Step 3/3: Landing PR #103 — Show widgets in the dashboard +============================================================ +PR #103 is mergeable (mergeStateStatus=CLEAN) + +Adding PR #103 to merge queue +$ gh pr merge 103 --squash +PR #103: in merge queue (0m) — polling in 120s + +PR #103 merged! + + + +Autoland plan · 3 steps · 3 done · 0 remaining + + ✅ 1. Merged | Land | #101 - Add the widget model + ✅ 2. Merged | Land | #102 - Expose widgets over the API + ✅ 3. Merged | Land | #103 - Show widgets in the dashboard + +All PRs landed successfully! + diff --git a/tests/golden/autoland/stack_merge_falls_through.plan.txt b/tests/golden/autoland/stack_merge_falls_through.plan.txt new file mode 100644 index 0000000..c2f2a6d --- /dev/null +++ b/tests/golden/autoland/stack_merge_falls_through.plan.txt @@ -0,0 +1,14 @@ +# Autoland plan — edit steps below. +# l [] = land that PR (a number, or its URL); a bare 'l' +# lands the next PR in the stack instead +# w = wait for a workflow to complete +# c [] = pause for manual confirmation; the optional +# condition names what to verify before proceeding +# (e.g. 'c QA sign-off complete') +# +# Lines starting with # are comments and are ignored. +# Blank lines are ignored. +# +l 101 # Add the widget model +l 102 # Expose widgets over the API +l 103 # Show widgets in the dashboard diff --git a/tests/golden/autoland/stack_merge_falls_through.status.json b/tests/golden/autoland/stack_merge_falls_through.status.json new file mode 100644 index 0000000..d6ce4c1 --- /dev/null +++ b/tests/golden/autoland/stack_merge_falls_through.status.json @@ -0,0 +1,53 @@ +{ + "status": "stopped", + "branch": "feat", + "base": "main", + "state_file": "/home/.stack-pr/autoland/feat.json", + "state_file_exists": true, + "lock_file": "/home/.stack-pr/autoland/feat.json.lock", + "pid": null, + "last_saved": "", + "abort_reason": null, + "resume_command": "stack-pr autoland --resume", + "plan": { + "total": 3, + "done": 3, + "remaining": 0, + "steps": [ + { + "number": 1, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 101, + "pr_url": "https://github.com/o/r/pull/101", + "title": "Add the widget model", + "detail": null + }, + { + "number": 2, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 102, + "pr_url": "https://github.com/o/r/pull/102", + "title": "Expose widgets over the API", + "detail": null + }, + { + "number": 3, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 103, + "pr_url": "https://github.com/o/r/pull/103", + "title": "Show widgets in the dashboard", + "detail": null + } + ] + }, + "other_runs": [] +} diff --git a/tests/golden/autoland/stack_merge_falls_through.status.txt b/tests/golden/autoland/stack_merge_falls_through.status.txt new file mode 100644 index 0000000..7f57be6 --- /dev/null +++ b/tests/golden/autoland/stack_merge_falls_through.status.txt @@ -0,0 +1,16 @@ +Autoland status + + Status: Stopped — not running, can be resumed + Branch: feat + Base: main + State file: /home/.stack-pr/autoland/feat.json + Last saved: + + +Autoland plan · 3 steps · 3 done · 0 remaining + + ✅ 1. Merged | Land | #101 - Add the widget model + ✅ 2. Merged | Land | #102 - Expose widgets over the API + ✅ 3. Merged | Land | #103 - Show widgets in the dashboard + +Resume with: stack-pr autoland --resume diff --git a/tests/golden/autoland/workflow_fails.checkpoint.json b/tests/golden/autoland/workflow_fails.checkpoint.json new file mode 100644 index 0000000..48ad909 --- /dev/null +++ b/tests/golden/autoland/workflow_fails.checkpoint.json @@ -0,0 +1,51 @@ +{ + "version": 1, + "branch": "feat", + "base": "main", + "plan_file": "/plan.autoland-plan", + "current_step": 1, + "last_landed_sha": "78e4ec1063c72debbb00f2ff966f72377789cdae", + "abort_reason": "Workflow deploy.yaml failed or timed out", + "stack": [ + { + "pr_url": "https://github.com/o/r/pull/101", + "pr_number": 101, + "branch": "TestBot/stack/101", + "title": "Add the widget model", + "review_decision": "APPROVED", + "state": "merged", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + }, + { + "pr_url": "https://github.com/o/r/pull/102", + "pr_number": 102, + "branch": "TestBot/stack/102", + "title": "Expose widgets over the API", + "review_decision": "APPROVED", + "state": "pending", + "check_retries": 0, + "queue_retries": 0, + "error_message": "" + } + ], + "plan": [ + { + "type": "land", + "entry_index": 0, + "pr_number": 101 + }, + { + "type": "workflow", + "workflow": "deploy.yaml", + "state": "failed", + "error_message": "Workflow timed out after 0h" + }, + { + "type": "land", + "entry_index": 1, + "pr_number": 102 + } + ] +} diff --git a/tests/golden/autoland/workflow_fails.out.txt b/tests/golden/autoland/workflow_fails.out.txt new file mode 100644 index 0000000..b97f19f --- /dev/null +++ b/tests/golden/autoland/workflow_fails.out.txt @@ -0,0 +1,56 @@ + +Discovering stack... + + +Autoland plan · 3 steps · 0 done · 3 remaining + +→ ⬜ 1. Pending | Land | #101 - Add the widget model + ⬜ 2. Pending | Workflow | deploy.yaml + ⬜ 3. Pending | Land | #102 - Expose widgets over the API +State file: /home/.stack-pr/autoland/feat.json + + +============================================================ +Step 1/3: Landing PR #101 — Add the widget model +============================================================ +PR #101 is mergeable (mergeStateStatus=CLEAN) + +Adding PR #101 to merge queue +$ gh pr merge 101 --squash +PR #101: in merge queue (0m) — polling in 120s + +PR #101 merged! + +Rebasing stack on origin/main... +$ git fetch origin main +$ git rebase origin/main +Re-submitting stack... + +============================================================ +Step 2/3: Workflow checkpoint — deploy.yaml +============================================================ + +Waiting for workflow: deploy.yaml +Target SHA: 78e4ec1063c7 +Workflow deploy.yaml: waiting (0m) — polling in 120s +Workflow deploy.yaml: waiting (2m) — polling in 120s +Workflow deploy.yaml: waiting (4m) — polling in 120s +Workflow deploy.yaml: waiting (6m) — polling in 120s +Workflow deploy.yaml: waiting (8m) — polling in 120s +Workflow deploy.yaml: waiting (10m) — polling in 120s + + + +Autoland plan · 3 steps · 1 done · 2 remaining + + ✅ 1. Merged | Land | #101 - Add the widget model +→ ❌ 2. Failed | Workflow | deploy.yaml + ↳ Workflow timed out after 0h + ⬜ 3. Pending | Land | #102 - Expose widgets over the API + +ABORTED: Workflow deploy.yaml failed or timed out + +Landing failed: Workflow deploy.yaml failed or timed out + +State saved to /home/.stack-pr/autoland/feat.json — resume with --resume + diff --git a/tests/golden/autoland/workflow_fails.plan.txt b/tests/golden/autoland/workflow_fails.plan.txt new file mode 100644 index 0000000..258a544 --- /dev/null +++ b/tests/golden/autoland/workflow_fails.plan.txt @@ -0,0 +1,14 @@ +# Autoland plan — edit steps below. +# l [] = land that PR (a number, or its URL); a bare 'l' +# lands the next PR in the stack instead +# w = wait for a workflow to complete +# c [] = pause for manual confirmation; the optional +# condition names what to verify before proceeding +# (e.g. 'c QA sign-off complete') +# +# Lines starting with # are comments and are ignored. +# Blank lines are ignored. +# +l 101 # Add the widget model +w deploy.yaml +l 102 # Expose widgets over the API diff --git a/tests/golden/autoland/workflow_fails.status.json b/tests/golden/autoland/workflow_fails.status.json new file mode 100644 index 0000000..fca9d25 --- /dev/null +++ b/tests/golden/autoland/workflow_fails.status.json @@ -0,0 +1,51 @@ +{ + "status": "stopped", + "branch": "feat", + "base": "main", + "state_file": "/home/.stack-pr/autoland/feat.json", + "state_file_exists": true, + "lock_file": "/home/.stack-pr/autoland/feat.json.lock", + "pid": null, + "last_saved": "", + "abort_reason": "Workflow deploy.yaml failed or timed out", + "resume_command": "stack-pr autoland --resume", + "plan": { + "total": 3, + "done": 1, + "remaining": 2, + "steps": [ + { + "number": 1, + "type": "land", + "outcome": "done", + "status": "Merged", + "is_next": false, + "pr_number": 101, + "pr_url": "https://github.com/o/r/pull/101", + "title": "Add the widget model", + "detail": null + }, + { + "number": 2, + "type": "workflow", + "outcome": "failed", + "status": "Failed", + "is_next": true, + "workflow": "deploy.yaml", + "detail": "Workflow timed out after 0h" + }, + { + "number": 3, + "type": "land", + "outcome": "pending", + "status": "Pending", + "is_next": false, + "pr_number": 102, + "pr_url": "https://github.com/o/r/pull/102", + "title": "Expose widgets over the API", + "detail": null + } + ] + }, + "other_runs": [] +} diff --git a/tests/golden/autoland/workflow_fails.status.txt b/tests/golden/autoland/workflow_fails.status.txt new file mode 100644 index 0000000..0aac19e --- /dev/null +++ b/tests/golden/autoland/workflow_fails.status.txt @@ -0,0 +1,19 @@ +Autoland status + + Status: Stopped — not running, can be resumed + Branch: feat + Base: main + State file: /home/.stack-pr/autoland/feat.json + Last saved: + + +Autoland plan · 3 steps · 1 done · 2 remaining + + ✅ 1. Merged | Land | #101 - Add the widget model +→ ❌ 2. Failed | Workflow | deploy.yaml + ↳ Workflow timed out after 0h + ⬜ 3. Pending | Land | #102 - Expose widgets over the API + +ABORTED: Workflow deploy.yaml failed or timed out + +Resume with: stack-pr autoland --resume diff --git a/tests/test_autoland_golden.py b/tests/test_autoland_golden.py new file mode 100644 index 0000000..fc9e787 --- /dev/null +++ b/tests/test_autoland_golden.py @@ -0,0 +1,269 @@ +"""Golden-file characterization of whole autoland runs. + +Each scenario runs `stack-pr autoland` end to end against a scripted GitHub +(see tests/autoland_scenarios.py) and compares what a user could observe with +the files in tests/golden/autoland/. Regenerate them with UPDATE_GOLDEN=1; see +the README there. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import pytest +from pytest_mock import MockerFixture + +from stack_pr.autoland.model import ConfirmStep, LandingContext, PRState, WorkflowStep +from stack_pr.autoland.plan import parse_plan +from stack_pr.autoland.state import AutolandCheckpointer +from tests.autoland_scenarios import ( + BRANCH, + FakePR, + FakeWorld, + Scenario, + run_scenario, + sha, + stack_entries, +) + +GOLDEN_DIR = Path(__file__).parent / "golden" / "autoland" +UPDATE = os.environ.get("UPDATE_GOLDEN") == "1" + + +def _prs(*numbers: int, **overrides: Any) -> list[FakePR]: + titles = { + 101: "Add the widget model", + 102: "Expose widgets over the API", + 103: "Show widgets in the dashboard", + } + return [FakePR(n, titles[n], **overrides) for n in numbers] + + +# --- checkpoints of earlier runs ------------------------------------------- + + +def _resume_checkpoint( + world: FakeWorld, _plan_file: Path +) -> tuple[AutolandCheckpointer, LandingContext]: + """A run started after #101 and #102 had landed (so the workflow and the + confirmation between them are assumed done), stopped when the workflow + after #102 timed out.""" + stack = stack_entries(world, [103]) + plan = parse_plan( + "l 101\nw deploy.yaml\nc QA sign-off complete\nl 102\nw deploy.yaml\nl 103\n", + stack, + pr_is_merged=lambda n: n in (101, 102), + ) + workflow = plan[4] + assert isinstance(workflow, WorkflowStep) + workflow.state = "failed" + workflow.error_message = "Workflow timed out after 3h" + ctx = LandingContext( + stack=stack, + plan=plan, + current_step=4, + aborted=True, + abort_reason="Workflow deploy.yaml failed or timed out", + last_landed_sha=sha("merge-102"), + ) + return AutolandCheckpointer(path=Path(), branch=BRANCH, base="main"), ctx + + +_OLD_REPLAN_PLAN = """\ +l +w deploy.yaml +c QA sign-off complete +l +w deploy.yaml +c metrics look healthy +c on-call agrees +l +""" + + +def _replan_checkpoint( + world: FakeWorld, plan_file: Path +) -> tuple[AutolandCheckpointer, LandingContext]: + """A run interrupted while landing #103, after #101 and #102 landed and + every checkpoint between them was done.""" + stack = stack_entries(world, [101, 102, 103]) + plan = parse_plan(_OLD_REPLAN_PLAN, stack, pr_is_merged=lambda _n: False) + for entry in stack[:2]: + entry.state = PRState.MERGED + stack[2].state = PRState.WAITING_FOR_CHECKS + stack[2].error_message = "Checks in progress..." + for step in plan[:7]: + if isinstance(step, WorkflowStep): + step.state = "succeeded" + elif isinstance(step, ConfirmStep): + step.confirmed = True + ctx = LandingContext( + stack=stack, + plan=plan, + current_step=7, + current_index=2, + aborted=True, + abort_reason="User interrupted (Ctrl+C)", + last_landed_sha=sha("merge-102"), + ) + checkpointer = AutolandCheckpointer( + path=Path(), branch=BRANCH, base="main", plan_file=plan_file + ) + return checkpointer, ctx + + +# --- the scenarios ----------------------------------------------------------- + + +def _one_at_a_time_prs() -> list[FakePR]: + first, second = _prs(101, 102) + first.checks = ["pending", "pass"] + second.review, second.approve_after = "REVIEW_REQUIRED", 4 + return [first, second] + + +SCENARIOS = [ + # Two PRs landed one at a time, with a workflow and a confirmation between. + Scenario( + name="one_at_a_time", + prs=_one_at_a_time_prs(), + stack=[101, 102], + plan="l\nw deploy.yaml\nc QA sign-off complete\nl\n", + answers=["n", "y"], + workflows={"deploy.yaml": "success"}, + ), + # A run of two land steps merges as one GitHub stack; the land step after + # the checkpoints is a run of one, so it lands on its own. + Scenario( + name="merge_as_stack", + prs=_prs(101, 102, 103), + stack=[101, 102, 103], + plan="l\nl\nw deploy.yaml\nc\nl\n", + config={"merge_as_stack": "true"}, + answers=["y"], + workflows={"deploy.yaml": "success"}, + ), + # The stack merge fails after landing only its bottom PR; the rest of the + # run lands one PR at a time. + Scenario( + name="stack_merge_falls_through", + prs=_prs(101, 102, 103), + stack=[101, 102, 103], + config={"merge_as_stack": "true"}, + stack_merge="fail", + stack_merge_lands=[101], + stack_merge_message="Merge conflict in PR #102", + ), + # Checks keep failing after their one rerun: the run stops at the first PR. + Scenario( + name="checks_fail", + prs=[ + FakePR(101, "Add the widget model", checks=["fail", "pending", "fail"]), + *_prs(102), + ], + stack=[101, 102], + expect_exit=1, + ), + # Every deploy run fails, so the workflow checkpoint times out. + Scenario( + name="workflow_fails", + prs=_prs(101, 102), + stack=[101, 102], + plan="l\nw deploy.yaml\nl\n", + workflows={"deploy.yaml": "failure"}, + expect_exit=1, + ), + # A confirmation with no terminal to answer it. + Scenario( + name="confirm_eof", + prs=_prs(101, 102), + stack=[101, 102], + plan="l\nc QA sign-off complete\nl\n", + expect_exit=1, + ), + # --resume after a workflow timed out: waits for it again, then lands the + # rest. The plan's already-landed prefix stays credited. + Scenario( + name="resume_mid_plan", + prs=_prs(101, 102, 103), + stack=[103], + flags=["--resume"], + workflows={"deploy.yaml": "success"}, + landed=[101, 102], + checkpoint=_resume_checkpoint, + ), + # A fresh run of a plan whose first PRs already landed: the steps up to + # the last of them are skipped, and assumed done. + Scenario( + name="landed_prefix", + prs=_prs(101, 102, 103), + stack=[103], + landed=[101, 102], + plan=( + "l 101\nw deploy.yaml\nc QA sign-off complete\nl 102\n" + "w deploy.yaml\nl 103\n" + ), + workflows={"deploy.yaml": "success"}, + ), + # --replan with an edited plan file: finished checkpoints after the same + # PRs carry over, a new one runs, and a removed one is reported. + Scenario( + name="replan_carry_over", + prs=_prs(101, 102, 103), + stack=[103], + plan=( + "l 101\nw deploy.yaml\nc QA sign-off complete\nl 102\n" + "w deploy.yaml\nc metrics look healthy\nc docs updated\nl 103\n" + ), + flags=["--replan"], + answers=["y", "y"], + workflows={"deploy.yaml": "success"}, + landed=[101, 102], + checkpoint=_replan_checkpoint, + ), +] + + +ARTIFACTS = { + "out.txt": "out", + "checkpoint.json": "checkpoint", + "status.txt": "status_text", + "status.json": "status_json", + "plan.txt": "plan", +} + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.name) +def test_golden_scenario( + scenario: Scenario, + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + tmp_path: Path, +) -> None: + result = run_scenario( + scenario, + mocker=mocker, + monkeypatch=monkeypatch, + capsys=capsys, + tmp_path=tmp_path, + ) + + assert result.exit_code == scenario.expect_exit + # The plan the editor shows parses back into the same plan. + assert result.plan_round_trip == result.plan + + for suffix, attr in ARTIFACTS.items(): + path = GOLDEN_DIR / f"{scenario.name}.{suffix}" + actual = getattr(result, attr) + if UPDATE: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(actual) + continue + assert path.exists(), f"missing {path.name}; generate it with UPDATE_GOLDEN=1" + assert actual == path.read_text(), ( + f"{path.name} differs; if the change is intended, regenerate with " + "UPDATE_GOLDEN=1 and review the diff" + )