From 98f9b85c3bc1fab14b425ce6f468743d6036b276 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 30 Oct 2026 01:47:56 +0000 Subject: [PATCH 01/70] feat(component): select lockless workflows per component Read lockless-components.txt beside the top-level azldev.toml during project configuration loading. Listed components use lockless behavior across all commands; unlisted components retain lock-file behavior. A missing or empty list selects locked behavior, while --without-lockfile overrides the list and makes every component lockless, including during rendering. Apply the selection to configuration merging and validation, lock loading and consistency checks, and synthetic history for render, build, and source preparation. Compare resolved project inputs for lockless components and stored lock fingerprints for locked components in component changed. Expose both update and refresh-upstream-commit for mixed projects. Each writer skips components using the other mode and preserves their state files while retaining true orphan detection. History and query skip lockless components, matching the existing global lockless no-op behavior. Add mixed-mode regression tests and update user documentation, generated CLI references, and emitted agent guidance for both workflows. Validation: affected unit test packages, mage fix all, and mage check all pass; mage docs succeeds. mage all fails on nine existing bumpspec tests. Scenario updates are blocked by enforcing SELinux and an unavailable container provider. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3497f2b2-c7fb-456e-a58f-6889b6ec67e2 --- docs/user/README.md | 2 +- docs/user/explanation/config-system.md | 2 +- docs/user/how-to/preview-without-lockfile.md | 87 +++++--- docs/user/reference/cli/azldev_component.md | 1 + .../reference/cli/azldev_component_changed.md | 5 + .../reference/cli/azldev_component_history.md | 2 + .../reference/cli/azldev_component_query.md | 2 + ...zldev_component_refresh-upstream-commit.md | 101 ++++++++++ .../reference/cli/azldev_component_render.md | 6 + .../reference/cli/azldev_component_update.md | 4 + .../cli/azldev_docs_agent_install.md | 5 +- docs/user/reference/config/components.md | 7 + docs/user/reference/config/config-file.md | 2 +- internal/app/azldev/agentskill/agentskill.go | 2 +- .../app/azldev/agentskill/agentskill_test.go | 16 +- .../azldev/agentskill/content/azldev.md.tmpl | 9 +- .../agentskill/content/comp-toml.md.tmpl | 7 + .../refresh-upstream-commit.md.tmpl | 7 +- .../content/update-component.md.tmpl | 5 + .../content/withoutlockfile/comp-toml.md.tmpl | 6 + internal/app/azldev/cmds/component/build.go | 2 +- internal/app/azldev/cmds/component/changed.go | 120 +++++++++-- .../cmds/component/changed_upstreamcommit.go | 10 +- .../changed_upstreamcommit_internal_test.go | 34 ++++ .../app/azldev/cmds/component/component.go | 7 +- .../azldev/cmds/component/component_test.go | 29 ++- internal/app/azldev/cmds/component/history.go | 9 +- .../app/azldev/cmds/component/lockmodes.go | 28 +++ .../app/azldev/cmds/component/preparerctx.go | 10 +- .../azldev/cmds/component/preparesources.go | 5 +- internal/app/azldev/cmds/component/query.go | 23 ++- .../cmds/component/refreshupstreamcommit.go | 21 +- .../component/refreshupstreamcommit_test.go | 53 ++++- internal/app/azldev/cmds/component/render.go | 30 ++- .../component/render_modes_internal_test.go | 186 ++++++++++++++++++ internal/app/azldev/cmds/component/update.go | 18 +- .../app/azldev/cmds/component/update_test.go | 35 ++++ internal/app/azldev/cmds/docs/agent.go | 5 +- .../app/azldev/core/components/resolver.go | 14 +- .../app/azldev/core/sources/sourceprep.go | 2 +- internal/app/azldev/env.go | 6 + internal/app/azldev/env_test.go | 18 ++ internal/lockfile/lockfile.go | 8 +- internal/lockfile/lockfile_test.go | 25 +++ internal/lockfile/show.go | 13 +- internal/lockfile/show_test.go | 13 ++ internal/lockfile/store.go | 7 +- internal/projectconfig/config.go | 18 +- internal/projectconfig/config_test.go | 87 +++++++- internal/projectconfig/configfile.go | 20 +- internal/projectconfig/loader.go | 38 ++-- internal/projectconfig/lockmodes.go | 52 +++++ internal/projectconfig/project.go | 33 ++-- scenario/without_lockfile_test.go | 46 +++-- 54 files changed, 1148 insertions(+), 155 deletions(-) create mode 100644 docs/user/reference/cli/azldev_component_refresh-upstream-commit.md rename internal/app/azldev/agentskill/content/{withoutlockfile => }/refresh-upstream-commit.md.tmpl (91%) create mode 100644 internal/app/azldev/cmds/component/lockmodes.go create mode 100644 internal/app/azldev/cmds/component/render_modes_internal_test.go create mode 100644 internal/projectconfig/lockmodes.go diff --git a/docs/user/README.md b/docs/user/README.md index cd9615ddb..e94e49b67 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -7,7 +7,7 @@ - [Build a Component](./how-to/build-component.md) — build RPMs from component definitions - [Build an Image](./how-to/build-image.md) — build and boot Azure Linux images - [Set Up AI Coding Agents](./how-to/set-up-ai-agents.md) — emit agent skill and instruction files -- [Preview the Lock-File-Free Mode](./how-to/preview-without-lockfile.md) — opt in to `--without-lockfile` +- [Preview the Lock-File-Free Mode](./how-to/preview-without-lockfile.md) — select lockless components or use `--without-lockfile` ## Explanation diff --git a/docs/user/explanation/config-system.md b/docs/user/explanation/config-system.md index ac5059ea9..abfe20462 100644 --- a/docs/user/explanation/config-system.md +++ b/docs/user/explanation/config-system.md @@ -99,7 +99,7 @@ Component definitions are merged additively. If the same component name (e.g., ` > **Note:** Slice fields (like `overlays`) are **appended**, not replaced, following the same merge behavior used by component configuration inheritance. -> **Preview:** With the global `--without-lockfile` flag, component definitions merge with override semantics instead, and component validation runs only after every included file has been merged, so an individual file may hold a partial component definition. See [Preview the Lock-File-Free Mode](../how-to/preview-without-lockfile.md). +> **Preview:** Components listed in `lockless-components.txt` beside the top-level `azldev.toml` merge with override semantics instead, and their validation runs only after every included file has been merged, so an individual file may hold a partial component definition. Unlisted components retain additive merging and per-file validation. The global `--without-lockfile` flag applies the lockless behavior to every component. See [Preview the Lock-File-Free Mode](../how-to/preview-without-lockfile.md). ### Component Groups and Images diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 15332b637..14f2fbc23 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -1,41 +1,71 @@ # How To: Preview the Lock-File-Free Mode -`--without-lockfile` is a **preview** global flag. It selects an alternative way of -tracking a component's resolved upstream commit: instead of per-component lock -files, azldev records the commit in generated component TOML that the project -includes like any other config file. +The **preview** lockless workflow tracks a component's resolved upstream commit +in generated component TOML instead of per-component lock files. Select it for +individual components using `lockless-components.txt`, or for every component +using the global `--without-lockfile` flag. -The flag is opt-in and defaults to off. Without it, azldev behaves exactly as it -always has — lock files, `component update`, `component history`, and -`component query` are unchanged. Nothing in the preview mode is stable yet; both -the command surface and the generated file layout may change. +The file applies to every command, not just rendering. Nothing in the preview +workflow is stable yet; both the command surface and generated layout may change. ```bash -# Default behavior: lock files. +# Use curl's selected workflow. azldev component render -p curl -# Preview behavior: generated upstream-commit config. -azldev --without-lockfile component render -p curl +# Refresh curl's generated commit if curl is listed in lockless-components.txt. +azldev component refresh-upstream-commit -p curl ``` -Pass the flag on every invocation that should use the preview mode, before the -command name. `--without-lockfile=false` explicitly selects the default mode. +The flag overrides the file: when set, every component is lockless, including +unlisted components. With the flag absent or set to `false`, the file selects +each component's mode. + +## Select Lock Mode Per Component + +Create `lockless-components.txt` beside the top-level `azldev.toml`. List one +exact, case-sensitive component name per line: + +```text +curl +rust-asyncband +``` + +Rendering, builds, and source preparation ignore existing locks and lock +validation for listed components. They use configured upstream commits, with +synthetic history derived from committed upstream-commit TOML changes. +Unlisted components use lock files, lock validation, and fingerprint-derived +synthetic history. Both kinds can be processed together. + +A missing or empty file means all components use lock files. Blank lines and +surrounding whitespace are ignored; names are not glob patterns. A file read +error stops configuration loading. + +Both `component update` and `component refresh-upstream-commit` are available +without the flag. Update (including `--bump`) skips listed components and leaves +their locks untouched; refresh skips unlisted components and preserves their +generated TOML files. All-component scans still prune files for components that +no longer exist, not files belonging to components using the other mode. +`component history` and `component query` skip listed components, matching the +existing lockless no-op behavior. `component changed` compares resolved project +inputs for listed components and stored fingerprints for unlisted components; +the current checkout's list determines which comparison to use. ## What Changes -| Area | Default | `--without-lockfile` | +| Area | Locked component | Lockless component | |------|---------|----------------------| | Resolved commit storage | `locks/.lock` | `base/upstream-commits/.toml` | | Refresh command | `azldev component update` | `azldev component refresh-upstream-commit` | | Inspecting resolved state | `azldev component history`, `azldev component query` | read the generated TOML; no equivalent commands | -| Lock consistency checks | On, with `--skip-lock-validation` to opt out | Not applicable; the flag is not registered | +| Lock consistency checks | On, with `--skip-lock-validation` to opt out | Skipped | | `component changed` | Compares stored input fingerprints | Compares project configuration resolved at each ref | | Synthetic dist-git history | Derived from lock-file fingerprint changes | Derived from generated upstream-commit TOML changes | -| Agent skills and MCP tools | Describe the lock-file workflow | Describe the upstream-commit workflow | +| Agent skills and MCP tools | Both workflows are available for mixed projects | The global flag selects the entirely lockless catalog | `component update`, `component history`, and `component query` remain registered in preview mode as hidden no-ops so that existing scripts report clearly that the -commands do nothing, rather than failing with "unknown command". +commands do nothing, rather than failing with "unknown command". Without the +global flag, they process only locked components. ## Configure the Project @@ -51,23 +81,24 @@ includes = [ Generated files hold only `spec.upstream-commit`; the component's own TOML supplies the source type and everything else. Because a single file may hold a -partial component definition in this mode, component validation runs after all -config files have been merged. +partial component definition for a lockless component, its validation runs after +all config files have been merged. Unlisted components retain normal additive +merging and per-file validation. -An existing `[project] lock-dir` setting is accepted and ignored in preview mode, -so the same project config works in both modes. +`[project] lock-dir` still configures the directory used for locked components. +Lockless components do not read or write their lock files. ## Refresh a Component ```bash # Resolve and record the upstream commit for one component. -azldev --without-lockfile component refresh-upstream-commit -p curl +azldev component refresh-upstream-commit -p curl # Refresh everything and prune generated files for components that no longer exist. -azldev --without-lockfile component refresh-upstream-commit -a +azldev component refresh-upstream-commit -a # CI gate: exit 1 when any generated file is out of date. -azldev --without-lockfile component refresh-upstream-commit -a --check-only -q +azldev component refresh-upstream-commit -a --check-only -q ``` Refresh after changing a commit pin, upstream distro or version, or snapshot. @@ -89,10 +120,10 @@ produce a synthetic commit. ## Detect Changed Components ```bash -azldev --without-lockfile component changed --from main -a -q -O json +azldev component changed --from main -a -q -O json ``` -In preview mode this loads the project configuration independently at both refs +For lockless components this loads the project configuration independently at both refs and compares the resolved component build inputs: normalized component configuration, upstream commit or local spec-directory contents, overlay source filenames and contents, and the effective distro release version. Documentation, @@ -101,8 +132,8 @@ fields do not mark a component as changed. ## Emit Agent Files for the Preview Mode -`azldev docs agent install` emits the content for the mode it runs in, so pass the -flag when the target repository uses the preview workflow: +`azldev docs agent install` includes both workflows for mixed projects. Pass the +flag to emit content for an entirely lockless project: ```bash azldev --without-lockfile docs agent install diff --git a/docs/user/reference/cli/azldev_component.md b/docs/user/reference/cli/azldev_component.md index df0e9622b..621d10121 100644 --- a/docs/user/reference/cli/azldev_component.md +++ b/docs/user/reference/cli/azldev_component.md @@ -50,6 +50,7 @@ components defined in the project configuration. * [azldev component metadata](azldev_component_metadata.md) - List documentation metadata for components' overlays and groups * [azldev component prepare-sources](azldev_component_prepare-sources.md) - Prepare buildable sources for components * [azldev component query](azldev_component_query.md) - Query info from locally rendered component specs +* [azldev component refresh-upstream-commit](azldev_component_refresh-upstream-commit.md) - Resolve and record upstream commits for components * [azldev component render](azldev_component_render.md) - Render post-overlay specs and sidecar files to a checked-in directory * [azldev component test](azldev_component_test.md) - Run a component's TMT tests in a local QEMU VM * [azldev component update](azldev_component_update.md) - Resolve and lock source identities for components diff --git a/docs/user/reference/cli/azldev_component_changed.md b/docs/user/reference/cli/azldev_component_changed.md index 2752c4ea2..581734e5d 100644 --- a/docs/user/reference/cli/azldev_component_changed.md +++ b/docs/user/reference/cli/azldev_component_changed.md @@ -12,6 +12,11 @@ input fingerprint changed (any change). For components whose fingerprint changed, it also reports whether the rendered sources file changed (sources change). +For components listed in lockless-components.txt beside the current checkout's +top-level azldev.toml, compare resolved project build inputs at both refs instead +of stored locks. The global --without-lockfile flag selects that comparison for +every component. + This is useful for CI/CD pipelines to determine which components need to be rebuilt or have their lookaside tarballs re-uploaded after a PR merge. diff --git a/docs/user/reference/cli/azldev_component_history.md b/docs/user/reference/cli/azldev_component_history.md index b269a82d5..8ac935060 100644 --- a/docs/user/reference/cli/azldev_component_history.md +++ b/docs/user/reference/cli/azldev_component_history.md @@ -15,6 +15,8 @@ Report three independent change-activity signals per component: Use this to find which packages get the most attention (for documentation, review prioritization, or refactoring planning). +Components using the lockless workflow via lockless-components.txt are skipped. + When a component shares its source TOML with other components (e.g., a bare entry in a shared components.toml), the toml-commit count is coarse and the component is marked 'toml-shared'. Use --shared=omit to drop those rows. diff --git a/docs/user/reference/cli/azldev_component_query.md b/docs/user/reference/cli/azldev_component_query.md index f840ca4cd..17f212c53 100644 --- a/docs/user/reference/cli/azldev_component_query.md +++ b/docs/user/reference/cli/azldev_component_query.md @@ -14,6 +14,8 @@ single shared mock chroot, batching all specs into one chroot invocation with parallel per-spec processing. For each component, it reports the source NEVR and the list of binary subpackages the spec would produce when built. +Components using the lockless workflow via lockless-components.txt are skipped. + The rendered-specs-dir must exist on disk; if it doesn't, run 'azldev component render' first. Components that previously failed to render (those with a RENDER_FAILED marker file) are skipped with a warning. diff --git a/docs/user/reference/cli/azldev_component_refresh-upstream-commit.md b/docs/user/reference/cli/azldev_component_refresh-upstream-commit.md new file mode 100644 index 000000000..aaebe323c --- /dev/null +++ b/docs/user/reference/cli/azldev_component_refresh-upstream-commit.md @@ -0,0 +1,101 @@ + + +## azldev component refresh-upstream-commit + +Resolve and record upstream commits for components + +### Synopsis + +Resolve upstream commits for components and write normal per-component TOML configuration. + +For upstream components, this resolves the effective commit hash using the +distro snapshot time, then records it as spec.upstream-commit in +base/upstream-commits/.toml by default. Include that directory's TOML +files before component-specific TOML configuration so subsequent commands use +the generated commit unless the component configuration explicitly overrides +it. + +Without --without-lockfile, only components listed in lockless-components.txt +beside the top-level azldev.toml are refreshed. Other components are skipped +and their generated TOML files are preserved. With the global flag, every +component uses the lockless workflow. + +All TOML configuration is resolved before the source type is checked. Selected +components whose effective spec.type is not "upstream" do not contact an +upstream provider, and any generated upstream-commit TOML for them is removed. +Configuration is loaded permissively for this command so stale generated pins +cannot prevent cleanup after a component is removed or changed to another +source type. Other configuration validation failures are reported as warnings. + +If an upstream component fails to resolve and '--check-only' is not set, +successful components are still written before the command returns an error. +Failed components are left unchanged, and orphan pruning is skipped for that +run. + +When updating all components (-a), orphan generated TOML files are +automatically pruned. +Orphan pruning is skipped when updating individual components to avoid +accidentally removing files for components not included in the filter. + +The --check-only flag runs the full pipeline but does NOT write TOML files or +prune orphans. The command exits 0 when nothing would change and exits 1 when +any component is stale or any generated TOML would be pruned. Intended for CI gates. + +``` +azldev component refresh-upstream-commit [flags] +``` + +### Examples + +``` + # Refresh all components + azldev component refresh-upstream-commit -a + + # Refresh a single component + azldev component refresh-upstream-commit -p curl + + # Refresh components in a group + azldev component refresh-upstream-commit -g core + + # Write generated files to a custom directory + azldev component refresh-upstream-commit -a --upstream-commits-dir config/commits + + # CI gate: exit 0 if commit TOMLs are current, 1 if anything would change + azldev component refresh-upstream-commit -a --check-only -q +``` + +### Options + +``` + -a, --all-components Include all components + --check-only resolve upstream commits but do not write TOML files or prune orphans. Exits 0 when nothing would change and 1 when any component is stale (or, with --all-components, when any orphan generated TOML would be pruned). Intended for CI gates + -p, --component stringArray Component name pattern + -g, --component-group stringArray Component group name + -h, --help help for refresh-upstream-commit + -s, --spec-path stringArray Spec path + --upstream-commits-dir string directory for generated per-component upstream-commit TOML files (default "base/upstream-commits") +``` + +### Options inherited from parent commands + +``` + -y, --accept-all accept all prompts + --color mode output colorization mode {always, auto, never} (default auto) + --concurrency int base concurrency limit (default: number of logical CPUs; minimum 1) + --config-file stringArray additional TOML config file(s) to merge (may be repeated) + -n, --dry-run dry run only (do not take action) + --network-retries int maximum number of attempts for network operations (minimum 1) (default 3) + --no-default-config disable default configuration + -O, --output-format fmt output format {csv, json, markdown, table} (default table) + --permissive-config do not fail on unknown fields in TOML config files + -C, --project string path to Azure Linux project + -q, --quiet only enable minimal output + --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) + -v, --verbose enable verbose output + --without-lockfile preview: track resolved upstream commits in generated config instead of lock files +``` + +### SEE ALSO + +* [azldev component](azldev_component.md) - Manage components + diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 88a9d5157..c1c3d1ca2 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -20,6 +20,12 @@ Unlike prepare-sources, render skips downloading source tarballs from the lookaside cache — only spec files, patches, scripts, and other git-tracked sidecar files are included. Multiple components can be rendered at once. +Render mode is selected per component by lockless-components.txt beside the +top-level azldev.toml. Each nonempty line names one component to render without +its lock file, using configured upstream commits and their TOML history instead. +All other components use lock files. A missing or empty file means all components +use lock files. The global --without-lockfile flag makes every component lockless. + When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to any component in the project config). Per-component dirs that ARE in config diff --git a/docs/user/reference/cli/azldev_component_update.md b/docs/user/reference/cli/azldev_component_update.md index 024d3a153..bd5d89e05 100644 --- a/docs/user/reference/cli/azldev_component_update.md +++ b/docs/user/reference/cli/azldev_component_update.md @@ -14,6 +14,10 @@ For local components, this computes a content hash of the spec directory. Subsequent commands (render, build) use the locked state for deterministic, reproducible results. +Components listed in lockless-components.txt beside the top-level azldev.toml +are skipped, including with --bump. Use refresh-upstream-commit for them. +Their existing lock files are left untouched. + If a component fails to resolve, successful components are still written before the command returns an error. Failed components are left unchanged, and orphan pruning is skipped for that run. diff --git a/docs/user/reference/cli/azldev_docs_agent_install.md b/docs/user/reference/cli/azldev_docs_agent_install.md index c8c8853a5..f6fd83574 100644 --- a/docs/user/reference/cli/azldev_docs_agent_install.md +++ b/docs/user/reference/cli/azldev_docs_agent_install.md @@ -31,8 +31,9 @@ MCP server. Directory paths in the emitted content (such as the lock, generated upstream-commit, and rendered-spec directories) are resolved from the loaded azldev.toml, falling back to azldev's built-in defaults when no configuration is -found. The emitted content also reflects the mode azldev runs in, so pass ---without-lockfile to describe the lock-file-free workflow. The bindings reflect +found. By default, the content includes both lock-file and lockless workflows +selected per component by lockless-components.txt. Pass --without-lockfile to +describe an entirely lockless project. The bindings reflect the project azldev runs in, so pair --output-dir with -C pointing at the target repository when scaffolding a different repo. diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index ce06e6f60..477f16a86 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -148,6 +148,13 @@ explicitly selected. `%autorelease` behavior is unchanged. The `[components..render]` section controls rendering behavior for a component. +Lock mode for every command is selected by `lockless-components.txt` beside the +top-level `azldev.toml`: listed components use the lockless workflow, and +unlisted components use lock files. A missing or empty list selects lock-file +behavior for every component. The global `--without-lockfile` flag overrides +the list and makes every component lockless. See +[per-component lock modes](../../how-to/preview-without-lockfile.md#select-lock-mode-per-component). + | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| | Skip file filter | `skip-file-filter` | boolean | No | Disable post-render file filtering (defaults to `false`) | diff --git a/docs/user/reference/config/config-file.md b/docs/user/reference/config/config-file.md index 258e9cf2b..1c24f7704 100644 --- a/docs/user/reference/config/config-file.md +++ b/docs/user/reference/config/config-file.md @@ -39,7 +39,7 @@ Glob patterns that match no files are silently ignored. Literal filenames (no wi Includes are resolved recursively — included files can themselves declare further includes. For a detailed explanation of load order and merge semantics, see [Configuration System](../../explanation/config-system.md). -> **Preview:** With the global `--without-lockfile` flag, the generated upstream-commit files must be included before any component-specific TOML configuration, so that later component definitions supply the remaining `spec` fields and may explicitly override a generated pin. See [Preview the Lock-File-Free Mode](../../how-to/preview-without-lockfile.md). +> **Preview:** For components listed in `lockless-components.txt` beside the top-level `azldev.toml`, or for every component with `--without-lockfile`, include generated upstream-commit files before component-specific TOML configuration so later definitions supply the remaining `spec` fields and may explicitly override a generated pin. See [Preview the Lock-File-Free Mode](../../how-to/preview-without-lockfile.md). ## Minimal Example diff --git a/internal/app/azldev/agentskill/agentskill.go b/internal/app/azldev/agentskill/agentskill.go index f700a8712..2f0ab5044 100644 --- a/internal/app/azldev/agentskill/agentskill.go +++ b/internal/app/azldev/agentskill/agentskill.go @@ -231,7 +231,7 @@ func (c Catalog) Skills() []Skill { result := slices.Clone(skills) if !c.withoutLockfile { - return result + return append(result, withoutLockfileSkills[updateComponentSkillName]) } for idx := range result { diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index a263a100f..ca143cd26 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -160,6 +160,16 @@ func TestComponentTomlSkillDocumentsRPMDevBumpspecPreview(t *testing.T) { assert.Contains(t, doc, "no bumpspec runtime discovery occurs") } +func TestComponentTomlSkillDocumentsPerComponentRenderModes(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev-comp-toml", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "`lockless-components.txt` beside the top-level `azldev.toml`") + assert.Contains(t, doc, "all others use lock files") + assert.Contains(t, doc, "for every command") + } +} + func TestImageSkillDocumentsSKUGroupGuidance(t *testing.T) { doc, err := agentskill.SkillDocument("azldev-image", testParams()) require.NoError(t, err) @@ -548,18 +558,18 @@ func TestImageSkillCoversCapabilities(t *testing.T) { } // TestCatalog_SkillsByMode verifies that each mode advertises the skill describing -// how it maintains resolved component state, and only that one. +// how it maintains resolved component state, including both workflows by default. func TestCatalog_SkillsByMode(t *testing.T) { defaultNames := skillNamesOf(agentskill.NewCatalog(false)) assert.Contains(t, defaultNames, "azldev-update-component") - assert.NotContains(t, defaultNames, "azldev-refresh-upstream-commit") + assert.Contains(t, defaultNames, "azldev-refresh-upstream-commit") withoutLockfileNames := skillNamesOf(agentskill.NewCatalog(true)) assert.Contains(t, withoutLockfileNames, "azldev-refresh-upstream-commit") assert.NotContains(t, withoutLockfileNames, "azldev-update-component") // The two catalogs otherwise describe the same set of skills, in the same order. - assert.Len(t, withoutLockfileNames, len(defaultNames)) + assert.Len(t, withoutLockfileNames, len(defaultNames)-1) } // TestCatalog_SkillContentByMode verifies that the rendered skill bodies describe diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index f3a572f86..c0c13e255 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -34,6 +34,11 @@ images assemble RPMs into bootable artifacts. ## Common commands +`lockless-components.txt` beside the top-level `azldev.toml` selects the lockless +workflow per component for all commands. Listed components ignore locks and use +generated upstream-commit TOML instead. All others use locks. The global +`--without-lockfile` flag makes every component lockless. + Top-level commands in this build: {{ range .TopLevelCommands }}- `azldev {{ .Name }}` — {{ .Short }} @@ -48,6 +53,7 @@ Everyday tasks (add `-q -O json` when scripting): | Build a component | `azldev comp build -p ` | | Render specs | `azldev comp render -p ` (or `-a`) | | Refresh a lock | `azldev comp update -p ` | +| Refresh a lockless component | `azldev comp refresh-upstream-commit -p ` | | List / build images | `azldev image list` / `azldev image build` | `comp` is an alias for `component`. Always confirm current syntax with @@ -65,6 +71,7 @@ mock helpers; it is intentionally omitted from normal help. - Build, iterate, and debug a component — read the `azldev-build-component` skill. - Build, boot, and register images — read the `azldev-image` skill. - Refresh a lock and finalize for a PR — read the `azldev-update-component` skill. +- Refresh a lockless component — read the `azldev-refresh-upstream-commit` skill. - Remove a component — read the `azldev-remove-component` skill. - Test built RPMs in a chroot — read the `azldev-mock` skill. @@ -72,7 +79,7 @@ mock helpers; it is intentionally omitted from normal help. - **Never edit generated output** — rendered specs and the output/work/log dirs are produced by azldev. Change the source config and re-render. -- **Re-run `azldev comp update` before opening a PR** — the lock fingerprint covers +- **For locked components, re-run `azldev comp update` before opening a PR** — the lock fingerprint covers the whole component config, and lock / rendered-spec CI checks run on committed state. - **Every overlay needs a `description`** explaining why the change is needed. diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 0def4bf59..fb7c3366d 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -96,6 +96,13 @@ used. `%autorelease` behavior is unchanged on either path. ## Render configuration +azldev reads `lockless-components.txt` beside the top-level `azldev.toml`. +Listed component names use the lockless workflow for every command; +all others use lock files. A missing or empty list means all components use +locks. The global `--without-lockfile` flag makes every component lockless. +Use `comp refresh-upstream-commit` for listed components and `comp update` +for unlisted components; each command skips the other mode. + `render.skip-file-filter = true` keeps all source and patch files during render. azldev normally prunes files not referenced by the rendered spec; set this only for the rare spec whose `Source`/`Patch` filenames use macros the filter cannot expand. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl similarity index 91% rename from internal/app/azldev/agentskill/content/withoutlockfile/refresh-upstream-commit.md.tmpl rename to internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index ecd6162d5..8309c8c56 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -40,6 +40,11 @@ Add `-O json` for machine-readable output when debugging. ## Finalizing component changes +Select this workflow by listing the component's exact name in +`lockless-components.txt` beside the top-level `azldev.toml`, or pass +`--without-lockfile` to make every component lockless. Without the flag, +refresh skips unlisted components and preserves their generated TOML files. + For edits that do not change source resolution, render and commit the result. The generated upstream commit TOML does not change: @@ -73,4 +78,4 @@ post-render amend: 5. `azldev comp render -p ` — `%changelog` / `Release:` now reflect the new commit TOML. 6. `git add {{ .RenderedSpecsDir }}///`, then `git commit --amend --no-edit` so the source change and rendered output land together. -Generated by `azldev docs agent`; do not hand-edit. Generated for azldev version `{{ .Version }}`. +Generated by `azldev docs agent`; do not hand-edit. diff --git a/internal/app/azldev/agentskill/content/update-component.md.tmpl b/internal/app/azldev/agentskill/content/update-component.md.tmpl index a8d158b77..cce79005a 100644 --- a/internal/app/azldev/agentskill/content/update-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/update-component.md.tmpl @@ -11,6 +11,11 @@ an input fingerprint computed from the component's render inputs — its TOML co overlays, the pinned upstream commit, and the distro release version. If any of those change, the lock is stale. +This workflow applies only to components not listed in `lockless-components.txt` +beside the top-level `azldev.toml`. Listed components use the +`azldev-refresh-upstream-commit` skill instead; `comp update` skips them. +The global `--without-lockfile` flag makes every component lockless. + ## When to run `update` | Situation | Run `update`? | diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 0e33a9e6d..42467b659 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -86,6 +86,12 @@ calculation = "manual" ## Render configuration +azldev reads `lockless-components.txt` beside the top-level `azldev.toml`. +Listed component names use the lockless workflow for every command; +all others use lock files. A missing or empty list means all components use +locks. The global `--without-lockfile` flag makes every component lockless. +Use `comp refresh-upstream-commit` for lockless components. + `render.skip-file-filter = true` keeps all source and patch files during render. azldev normally prunes files not referenced by the rendered spec; set this only for the rare spec whose `Source`/`Patch` filenames use macros the filter cannot expand. diff --git a/internal/app/azldev/cmds/component/build.go b/internal/app/azldev/cmds/component/build.go index 2c9f99171..3fff7e6c4 100644 --- a/internal/app/azldev/cmds/component/build.go +++ b/internal/app/azldev/cmds/component/build.go @@ -275,7 +275,7 @@ func buildComponent( var preparerOpts []sources.PreparerOption if !options.WithoutGitRepo { - preparerOpts = append(preparerOpts, gitRepoPreparerOptions(env, distro)...) + preparerOpts = append(preparerOpts, gitRepoPreparerOptions(env, distro, component.GetName())...) } if options.RPMDevBumpspec { diff --git a/internal/app/azldev/cmds/component/changed.go b/internal/app/azldev/cmds/component/changed.go index 31226191b..c70bbcb9a 100644 --- a/internal/app/azldev/cmds/component/changed.go +++ b/internal/app/azldev/cmds/component/changed.go @@ -67,6 +67,11 @@ input fingerprint changed (any change). For components whose fingerprint changed, it also reports whether the rendered sources file changed (sources change). +For components listed in lockless-components.txt beside the current checkout's +top-level azldev.toml, compare resolved project build inputs at both refs instead +of stored locks. The global --without-lockfile flag selects that comparison for +every component. + This is useful for CI/CD pipelines to determine which components need to be rebuilt or have their lookaside tarballs re-uploaded after a PR merge. @@ -140,10 +145,8 @@ const ( changeTypeDeleted = "deleted" ) -// ChangedComponents determines which components changed between two git refs, -// using the comparison that matches the active mode: stored lock file -// fingerprints by default, or configuration resolved at each ref when the global -// '--without-lockfile' flag is set. +// ChangedComponents compares stored fingerprints for locked components and +// resolved project inputs for lockless components. func ChangedComponents( env *azldev.Env, options *ChangedComponentOptions, ) ([]ChangedResult, error) { @@ -151,7 +154,63 @@ func ChangedComponents( return changedComponentsFromProjectConfigs(env, options) } - return changedComponentsFromLocks(env, options) + if !hasLocklessChangeSelection(env, &options.ComponentFilter) { + return changedComponentsFromLocks(env, options) + } + + locklessResults, err := changedComponentsFromProjectConfigs(env, options) + if err != nil { + return nil, err + } + + lockedOptions := *options + + lockedOptions.ComponentFilter.ComponentNamePatterns = nil + for _, pattern := range options.ComponentFilter.ComponentNamePatterns { + if !env.ComponentWithoutLockfile(pattern) { + lockedOptions.ComponentFilter.ComponentNamePatterns = append( + lockedOptions.ComponentFilter.ComponentNamePatterns, pattern, + ) + } + } + + if lockedOptions.ComponentFilter.HasNoCriteria() { + return locklessResults, nil + } + + lockedResults, err := changedComponentsFromLocks(env, &lockedOptions) + if err != nil { + return nil, err + } + + lockedResults = append(lockedResults, locklessResults...) + sort.Slice(lockedResults, func(i, j int) bool { return lockedResults[i].Component < lockedResults[j].Component }) + + return lockedResults, nil +} + +func hasLocklessChangeSelection(env *azldev.Env, filter *components.ComponentFilter) bool { + if len(env.Config().LocklessComponents) == 0 { + return false + } + + if filter.IncludeAllComponents || len(filter.ComponentGroupNames) > 0 || len(filter.SpecPaths) > 0 { + return true + } + + for name, lockless := range env.Config().LocklessComponents { + if !lockless { + continue + } + + for _, pattern := range filter.ComponentNamePatterns { + if matched, err := filepath.Match(pattern, name); matched || err != nil { + return true + } + } + } + + return false } // changedComponentsFromLocks compares component lock files and rendered sources @@ -163,13 +222,15 @@ func changedComponentsFromLocks( // the current working-tree locks may legitimately be stale. options.ComponentFilter.SkipLockValidation = true - resolver := components.NewResolver(env) - - comps, err := resolver.FindComponents(&options.ComponentFilter) + locked, err := findLockedComponentsForChanges(env, &options.ComponentFilter) if err != nil { return nil, fmt.Errorf("resolving components:\n%w", err) } + if locked.Len() == 0 && !options.ComponentFilter.IncludeAllComponents { + return []ChangedResult{}, nil + } + ctx, err := newChangedContext(env) if err != nil { return nil, err @@ -186,12 +247,12 @@ func changedComponentsFromLocks( } // Batch-read all locks at both refs. - fromLocks, err := lockfile.ReadAllAtCommit(ctx.repo, fromHash, ctx.lockRelDir) + fromLocks, err := readChangedLocks(env, ctx, fromHash) if err != nil { return nil, fmt.Errorf("reading locks at --from:\n%w", err) } - toLocks, err := lockfile.ReadAllAtCommit(ctx.repo, toHash, ctx.lockRelDir) + toLocks, err := readChangedLocks(env, ctx, toHash) if err != nil { return nil, fmt.Errorf("reading locks at --to:\n%w", err) } @@ -208,7 +269,7 @@ func changedComponentsFromLocks( } results, err := buildResults( - comps, fromLocks, toLocks, ctx, fromTree, toTree, + locked, fromLocks, toLocks, ctx, fromTree, toTree, options.IncludeUnchanged, options.ComponentFilter.IncludeAllComponents, ) if err != nil { @@ -218,6 +279,43 @@ func changedComponentsFromLocks( return results, nil } +func findLockedComponentsForChanges( + env *azldev.Env, filter *components.ComponentFilter, +) (*components.ComponentSet, error) { + resolver := components.NewResolver(env) + + comps, err := resolver.FindComponents(filter) + if err != nil { + return nil, fmt.Errorf("resolving components:\n%w", err) + } + + locked := components.NewComponentSet() + for _, comp := range componentsForLockMode(env, comps.Components(), false) { + locked.Add(comp) + } + + return locked, nil +} + +func readChangedLocks( + env *azldev.Env, ctx *changedContext, hash string, +) (map[string]lockfile.ComponentLock, error) { + var ignored []string + + for name, lockless := range env.Config().LocklessComponents { + if lockless { + ignored = append(ignored, name) + } + } + + locks, err := lockfile.ReadAllAtCommit(ctx.repo, hash, ctx.lockRelDir, ignored...) + if err != nil { + return nil, fmt.Errorf("reading historical component locks:\n%w", err) + } + + return locks, nil +} + // changedContext holds resolved repo state for change detection. type changedContext struct { repo *gogit.Repository diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit.go b/internal/app/azldev/cmds/component/changed_upstreamcommit.go index ce8e19ebc..426a3f8f2 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit.go @@ -84,6 +84,10 @@ func changedComponentsFromProjectConfigs( return nil, fmt.Errorf("selecting components:\n%w", err) } + names = slices.DeleteFunc(names, func(name string) bool { + return !env.ComponentWithoutLockfile(name) + }) + return buildHistoricalResults( names, fromProject, toProject, fromTree, toTree, options.IncludeUnchanged, options.ComponentFilter.IncludeAllComponents, @@ -176,7 +180,7 @@ func loadHistoricalProject( } if err := populateHistoricalComponents( - result, snapshotFS, historicalEnv, resolvedComponents, + result, snapshotFS, historicalEnv, resolvedComponents, env, ); err != nil { return nil, err } @@ -224,9 +228,13 @@ func populateHistoricalComponents( fs opctx.FS, env *azldev.Env, resolvedComponents *components.ComponentSet, + selectionEnv *azldev.Env, ) error { for _, component := range resolvedComponents.Components() { project.components[component.GetName()] = *component.GetConfig() + if !selectionEnv.ComponentWithoutLockfile(component.GetName()) { + continue + } inputs, err := buildComponentComparisonInputs(fs, env, component.GetConfig()) if err != nil { diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go index 357f3321b..f7b6795a7 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go @@ -268,6 +268,40 @@ upstream-commit = "abcdef1234567" assert.Equal(t, []string{"curl"}, project.componentGroups["core"]) } +func TestLoadHistoricalProject_OnlyBuildsInputsForLocklessComponents(t *testing.T) { + rootConfig := []byte(` +[project] +default-distro = { name = "testdistro", version = "1.0" } +rendered-specs-dir = "SPECS" + +[distros.testdistro] +description = "Test distro" +[distros.testdistro.versions."1.0"] +release-ver = "1.0" + +[components.rust-asyncband.spec] +type = "upstream" +upstream-commit = "abcdef1234567" + +[components.curl.spec] +type = "local" +path = "unavailable/curl.spec" +`) + repo, hashes := testRepoWithCommits(t, []testRepoCommit{ + {files: map[string][]byte{"azldev.toml": rootConfig}}, + }) + tree, err := resolveTree(repo, hashes[0]) + require.NoError(t, err) + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + + project, err := loadHistoricalProject(env.Env, tree, ".") + require.NoError(t, err, "locked component inputs must not be read for the lockless comparison") + assert.Contains(t, project.components, "curl") + assert.NotContains(t, project.comparisonInputs, "curl") + assert.Equal(t, "abcdef1234567", project.comparisonInputs["rust-asyncband"].SourceIdentity) +} + func TestSelectHistoricalComponentNames_UsesBothRefs(t *testing.T) { fromProject := &historicalProject{ components: map[string]projectconfig.ComponentConfig{ diff --git a/internal/app/azldev/cmds/component/component.go b/internal/app/azldev/cmds/component/component.go index dd75606fd..6cca90609 100644 --- a/internal/app/azldev/cmds/component/component.go +++ b/internal/app/azldev/cmds/component/component.go @@ -37,10 +37,8 @@ components defined in the project configuration.`, testOnAppInit(app, cmd) metadataOnAppInit(app, cmd) - // The commands that maintain resolved component state differ by mode: the - // default mode maintains lock files, while lock-file-free mode maintains - // generated upstream-commit config. Registering only the commands that - // belong to the active mode keeps help, docs, and MCP tools honest. + // Mixed projects need both state writers. The global override keeps the + // existing entirely lockless command surface and legacy no-ops. if app.WithoutLockfile() { legacyOnAppInit(app, cmd) refreshUpstreamCommitOnAppInit(app, cmd) @@ -48,6 +46,7 @@ components defined in the project configuration.`, historyOnAppInit(app, cmd) queryOnAppInit(app, cmd) updateOnAppInit(app, cmd) + refreshUpstreamCommitOnAppInit(app, cmd) } return cmd diff --git a/internal/app/azldev/cmds/component/component_test.go b/internal/app/azldev/cmds/component/component_test.go index 156fe6b36..c8ae56ebc 100644 --- a/internal/app/azldev/cmds/component/component_test.go +++ b/internal/app/azldev/cmds/component/component_test.go @@ -9,7 +9,10 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx/opctx_test" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -34,7 +37,7 @@ func TestOnAppInit_CommandsByMode(t *testing.T) { args []string refreshExpected bool }{ - {name: "default mode", args: nil, refreshExpected: false}, + {name: "default mode", args: nil, refreshExpected: true}, {name: "lock-file-free mode", args: []string{"--without-lockfile"}, refreshExpected: true}, } @@ -73,6 +76,28 @@ func TestNewComponentCommands_LockValidationFlagByMode(t *testing.T) { withoutLockfile := component.WithoutLockfileFlags() assert.Nil(t, component.NewBuildCmd(withoutLockfile).Flags().Lookup("skip-lock-validation")) - assert.Nil(t, component.NewRenderCmd(withoutLockfile).Flags().Lookup("skip-lock-validation")) + assert.NotNil(t, component.NewRenderCmd(withoutLockfile).Flags().Lookup("skip-lock-validation"), + "render selects lock behavior per component, independently of the global mode") assert.Nil(t, component.NewComponentListCommand(withoutLockfile).Flags().Lookup("skip-lock-validation")) } + +func TestLocklessComponents_SkipLegacyHistoryAndQuery(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + env.Config.Components["rust-asyncband"] = projectconfig.ComponentConfig{ + Name: "rust-asyncband", + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + } + filter := components.ComponentFilter{IncludeAllComponents: true} + + history, err := component.ComponentHistory(env.Env, &component.HistoryOptions{ + ComponentFilter: filter, + SharedTomlMode: "show", + }) + require.NoError(t, err) + assert.Empty(t, history) + + query, err := component.QueryComponents(env.Env, &component.QueryComponentsOptions{ComponentFilter: filter}) + require.NoError(t, err) + assert.Empty(t, query) +} diff --git a/internal/app/azldev/cmds/component/history.go b/internal/app/azldev/cmds/component/history.go index 739d1c2c1..274554aa4 100644 --- a/internal/app/azldev/cmds/component/history.go +++ b/internal/app/azldev/cmds/component/history.go @@ -63,6 +63,8 @@ func NewHistoryCmd() *cobra.Command { Use this to find which packages get the most attention (for documentation, review prioritization, or refactoring planning). +Components using the lockless workflow via lockless-components.txt are skipped. + When a component shares its source TOML with other components (e.g., a bare entry in a shared components.toml), the toml-commit count is coarse and the component is marked 'toml-shared'. Use --shared=omit to drop those rows. @@ -295,6 +297,11 @@ func ComponentHistory(env *azldev.Env, options *HistoryOptions) ([]HistoryResult return nil, fmt.Errorf("resolving components:\n%w", err) } + lockedComps := componentsForLockMode(env, comps.Components(), false) + if len(lockedComps) == 0 { + return []HistoryResult{}, nil + } + ctx, err := newHistoryContext(env) if err != nil { return nil, err @@ -308,7 +315,7 @@ func ComponentHistory(env *azldev.Env, options *HistoryOptions) ([]HistoryResult explicit := hasExplicitComponentSelection(&options.ComponentFilter) effectiveIncludeBare := options.IncludeBare || explicit - stubs := buildHistoryStubs(env, comps.Components(), effectiveIncludeBare) + stubs := buildHistoryStubs(env, lockedComps, effectiveIncludeBare) if len(stubs) == 0 { return nil, nil } diff --git a/internal/app/azldev/cmds/component/lockmodes.go b/internal/app/azldev/cmds/component/lockmodes.go new file mode 100644 index 000000000..7da2dfde6 --- /dev/null +++ b/internal/app/azldev/cmds/component/lockmodes.go @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "log/slog" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" +) + +func componentsForLockMode( + env *azldev.Env, comps []components.Component, lockless bool, +) []components.Component { + selected := make([]components.Component, 0, len(comps)) + for _, comp := range comps { + if env.ComponentWithoutLockfile(comp.GetName()) == lockless { + selected = append(selected, comp) + } + } + + if skipped := len(comps) - len(selected); skipped > 0 { + slog.Info("Skipping components using the other lock mode", "count", skipped) + } + + return selected +} diff --git a/internal/app/azldev/cmds/component/preparerctx.go b/internal/app/azldev/cmds/component/preparerctx.go index f98f7b671..10fdab680 100644 --- a/internal/app/azldev/cmds/component/preparerctx.go +++ b/internal/app/azldev/cmds/component/preparerctx.go @@ -17,9 +17,15 @@ import ( // is derived from the component's generated upstream-commit TOML, which has no // fingerprint to compare against, so only committed changes are represented. func gitRepoPreparerOptions( - env *azldev.Env, distro sourceproviders.ResolvedDistro, + env *azldev.Env, distro sourceproviders.ResolvedDistro, componentName string, ) []sources.PreparerOption { - if env.WithoutLockfile() { + return gitRepoPreparerOptionsForMode(env, distro, env.ComponentWithoutLockfile(componentName)) +} + +func gitRepoPreparerOptionsForMode( + env *azldev.Env, distro sourceproviders.ResolvedDistro, withoutLockfile bool, +) []sources.PreparerOption { + if withoutLockfile { return []sources.PreparerOption{ sources.WithGitRepo(env, nil, ""), sources.WithoutLockfileHistory(), diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index 317b7fb38..d55781037 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -141,7 +141,7 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er "synthetic history requires overlays to be applied") } - preparerOpts := buildPreparerOptions(env, distro, options) + preparerOpts := buildPreparerOptions(env, distro, options, component.GetName()) preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, append( preparerOpts, @@ -165,11 +165,12 @@ func buildPreparerOptions( env *azldev.Env, distro sourceproviders.ResolvedDistro, options *PrepareSourcesOptions, + componentName string, ) []sources.PreparerOption { var opts []sources.PreparerOption if !options.WithoutGitRepo && !options.SkipOverlays { - opts = append(opts, gitRepoPreparerOptions(env, distro)...) + opts = append(opts, gitRepoPreparerOptions(env, distro, componentName)...) } if options.RPMDevBumpspec { diff --git a/internal/app/azldev/cmds/component/query.go b/internal/app/azldev/cmds/component/query.go index 83014b12c..000529df9 100644 --- a/internal/app/azldev/cmds/component/query.go +++ b/internal/app/azldev/cmds/component/query.go @@ -53,6 +53,8 @@ single shared mock chroot, batching all specs into one chroot invocation with parallel per-spec processing. For each component, it reports the source NEVR and the list of binary subpackages the spec would produce when built. +Components using the lockless workflow via lockless-components.txt are skipped. + The rendered-specs-dir must exist on disk; if it doesn't, run 'azldev component render' first. Components that previously failed to render (those with a RENDER_FAILED marker file) are skipped with a warning.`, @@ -110,6 +112,18 @@ type componentDetails struct { func QueryComponents( env *azldev.Env, options *QueryComponentsOptions, ) ([]*componentDetails, error) { + resolver := components.NewResolver(env) + comps, resolveErr := resolver.FindComponents(&options.ComponentFilter) + + // Preserve directory diagnostics unless the selection is entirely lockless. + var lockedComps []components.Component + if resolveErr == nil { + lockedComps = componentsForLockMode(env, comps.Components(), false) + if comps.Len() > 0 && len(lockedComps) == 0 { + return []*componentDetails{}, nil + } + } + renderedSpecsDir := env.Config().Project.RenderedSpecsDir if renderedSpecsDir == "" { return nil, errors.New( @@ -128,11 +142,8 @@ func QueryComponents( renderedSpecsDir) } - resolver := components.NewResolver(env) - - comps, err := resolver.FindComponents(&options.ComponentFilter) - if err != nil { - return nil, fmt.Errorf("failed to resolve components:\n%w", err) + if resolveErr != nil { + return nil, fmt.Errorf("failed to resolve components:\n%w", resolveErr) } cfg := env.Config() @@ -142,7 +153,7 @@ func QueryComponents( "please use command-line options to indicate which components to query") } - inputs, _, err := buildSpecQueryInputs(env, comps.Components(), renderedSpecsDir) + inputs, _, err := buildSpecQueryInputs(env, lockedComps, renderedSpecsDir) if err != nil { return nil, err } diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit.go b/internal/app/azldev/cmds/component/refreshupstreamcommit.go index 02da3bddf..a1be4f70a 100644 --- a/internal/app/azldev/cmds/component/refreshupstreamcommit.go +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit.go @@ -54,6 +54,11 @@ files before component-specific TOML configuration so subsequent commands use the generated commit unless the component configuration explicitly overrides it. +Without --without-lockfile, only components listed in lockless-components.txt +beside the top-level azldev.toml are refreshed. Other components are skipped +and their generated TOML files are preserved. With the global flag, every +component uses the lockless workflow. + All TOML configuration is resolved before the source type is checked. Selected components whose effective spec.type is not "upstream" do not contact an upstream provider, and any generated upstream-commit TOML for them is removed. @@ -75,19 +80,19 @@ The --check-only flag runs the full pipeline but does NOT write TOML files or prune orphans. The command exits 0 when nothing would change and exits 1 when any component is stale or any generated TOML would be pruned. Intended for CI gates.`, Example: ` # Refresh all components - azldev --without-lockfile component refresh-upstream-commit -a + azldev component refresh-upstream-commit -a # Refresh a single component - azldev --without-lockfile component refresh-upstream-commit -p curl + azldev component refresh-upstream-commit -p curl # Refresh components in a group - azldev --without-lockfile component refresh-upstream-commit -g core + azldev component refresh-upstream-commit -g core # Write generated files to a custom directory - azldev --without-lockfile component refresh-upstream-commit -a --upstream-commits-dir config/commits + azldev component refresh-upstream-commit -a --upstream-commits-dir config/commits # CI gate: exit 0 if commit TOMLs are current, 1 if anything would change - azldev --without-lockfile component refresh-upstream-commit -a --check-only -q`, + azldev component refresh-upstream-commit -a --check-only -q`, RunE: azldev.RunFuncWithExtraArgs(func(env *azldev.Env, args []string) (interface{}, error) { options.ComponentFilter.ComponentNamePatterns = append( args, options.ComponentFilter.ComponentNamePatterns..., @@ -134,6 +139,8 @@ func RefreshUpstreamCommits( env *azldev.Env, options *RefreshUpstreamCommitOptions, ) ([]RefreshUpstreamCommitResult, error) { resolver := components.NewResolver(env) + options.ComponentFilter.SkipLockValidation = true + resolver.SuppressLockWarnings = true resolved, err := resolver.FindComponents(&options.ComponentFilter) if err != nil { @@ -164,7 +171,9 @@ func RefreshUpstreamCommits( // selected component has an upstream identity. Non-upstream components // never contact a provider; an existing generated pin is marked for // removal instead. - upstreamComps, results, inspectErr := inspectSelectedComponents(allComps, store) + upstreamComps, results, inspectErr := inspectSelectedComponents( + componentsForLockMode(env, allComps, true), store, + ) if inspectErr != nil { return results, inspectErr } diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go index c3e2de99b..9f42595a0 100644 --- a/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go @@ -55,12 +55,22 @@ func TestRefreshUpstreamCommitCmd_CleansPinsThatInvalidateStrictConfig(t *testin testCases := []struct { name string componentConfig string + withoutLockfile bool }{ { - name: "removed component", + name: "removed component", + withoutLockfile: true, }, { - name: "converted to local", + name: "converted to local", + withoutLockfile: true, + componentConfig: `[components.test-component.spec] +type = "local" +path = "../../specs/test-component.spec" +`, + }, + { + name: "listed component converted to local", componentConfig: `[components.test-component.spec] type = "local" path = "../../specs/test-component.spec" @@ -93,6 +103,11 @@ release-ver = "1.0" fileperms.PublicFile, )) + if !testCase.withoutLockfile { + require.NoError(t, fileutils.WriteFile(testEnv.TestFS, "/project/lockless-components.txt", + []byte("test-component\n"), fileperms.PublicFile)) + } + if testCase.componentConfig != "" { require.NoError(t, fileutils.WriteFile( testEnv.TestFS, @@ -110,10 +125,9 @@ release-ver = "1.0" testEnv.TestInterfaces.OSEnvFactory, ) - // The command is only registered in lock-file-free mode, which the - // CLI selects by pre-parsing the global flags before registration. - args := []string{ - "--without-lockfile", "component", "refresh-upstream-commit", "--all-components", + args := []string{"component", "refresh-upstream-commit", "--all-components"} + if testCase.withoutLockfile { + args = append([]string{"--without-lockfile"}, args...) } app.PreParseGlobalFlags(args) @@ -168,6 +182,33 @@ func TestRefreshUpstreamCommits_WritesCommit(t *testing.T) { assert.Equal(t, commit, savedCommit) } +func TestRefreshUpstreamCommits_OnlyRefreshesLocklessComponents(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + setupMockGit(env, "new-commit") + addRefreshUpstreamComponent(env, "curl") + addRefreshUpstreamComponent(env, "rust-asyncband") + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + require.NoError(t, store.Save("curl", "preserved-commit")) + + results, err := componentcmds.RefreshUpstreamCommits( + env.Env, &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + }) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "rust-asyncband", results[0].Component) + + commit, exists, err := store.Get("rust-asyncband") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "new-commit", commit) + commit, exists, err = store.Get("curl") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "preserved-commit", commit) +} + func TestRefreshUpstreamCommits_ConfigOnlyChangeDoesNotChangeCommitTOML(t *testing.T) { env := testutils.NewTestEnvWithoutLockfile(t) diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index ff99942bd..c9ad8a758 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -41,12 +41,12 @@ type RenderOptions struct { RPMDevBumpspec bool } -func renderOnAppInit(app *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewRenderCmd(cmdOptionsForApp(app)...)) +func renderOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewRenderCmd()) } // NewRenderCmd constructs a [cobra.Command] for the "component render" CLI subcommand. -func NewRenderCmd(opts ...CmdOption) *cobra.Command { +func NewRenderCmd(_ ...CmdOption) *cobra.Command { var options RenderOptions var cmd *cobra.Command @@ -68,6 +68,12 @@ Unlike prepare-sources, render skips downloading source tarballs from the lookaside cache — only spec files, patches, scripts, and other git-tracked sidecar files are included. Multiple components can be rendered at once. +Render mode is selected per component by lockless-components.txt beside the +top-level azldev.toml. Each nonempty line names one component to render without +its lock file, using configured upstream commits and their TOML history instead. +All other components use lock files. A missing or empty file means all components +use lock files. The global --without-lockfile flag makes every component lockless. + When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to any component in the project config). Per-component dirs that ARE in config @@ -101,8 +107,13 @@ valid with -a.`, }, } - addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) + addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions{}) + addRenderOptions(cmd, &options) + + return cmd +} +func addRenderOptions(cmd *cobra.Command, options *RenderOptions) { cmd.Flags().StringVarP(&options.OutputDir, "output-dir", "o", "", "output directory for rendered specs (overrides rendered-specs-dir from config)") _ = cmd.MarkFlagDirname("output-dir") @@ -129,8 +140,6 @@ valid with -a.`, // muddled (CI would fail on stale failures even when on-disk markers // already record them) and forcing a choice keeps the contract crisp. cmd.MarkFlagsMutuallyExclusive("fail-on-error", "check-only") - - return cmd } // RenderResult holds the result of rendering a single component. @@ -412,7 +421,8 @@ func parallelPrepare( //nolint:contextcheck // env carries the ctx return prepareOneComponent( workerEnv, mockProcessor, comp, stagingDir, options.OutputDir, - options.specEditorMode(), options.RPMDevBumpspec) + options.specEditorMode(), options.RPMDevBumpspec, + env.ComponentWithoutLockfile(comp.GetName())) }, ) @@ -461,6 +471,7 @@ func prepareOneComponent( outputDir string, specEditor spec.EditorMode, rpmDevBumpspec bool, + lockless bool, ) prepResult { componentName := comp.GetName() @@ -475,7 +486,7 @@ func prepareOneComponent( }} } - prep, err := prepareComponentSources(env, mockProcessor, comp, stagingDir, specEditor, rpmDevBumpspec) + prep, err := prepareComponentSources(env, mockProcessor, comp, stagingDir, specEditor, rpmDevBumpspec, lockless) if err != nil { slog.Error("Failed to prepare component sources", "component", componentName, "error", err) @@ -503,6 +514,7 @@ func prepareComponentSources( stagingDir string, specEditor spec.EditorMode, rpmDevBumpspec bool, + lockless bool, ) (*preparedComponent, error) { componentName := comp.GetName() @@ -533,7 +545,7 @@ func prepareComponentSources( // rpmautospec can expand %autorelease and %autochangelog correctly. // WithSkipLookaside avoids expensive tarball downloads — only spec + // sidecar files are needed for rendering. - preparerOpts := append(gitRepoPreparerOptions(env, distro), + preparerOpts := append(gitRepoPreparerOptionsForMode(env, distro, lockless), sources.WithSkipLookaside(), sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), sources.WithMockProcessor(mockProcessor), diff --git a/internal/app/azldev/cmds/component/render_modes_internal_test.go b/internal/app/azldev/cmds/component/render_modes_internal_test.go new file mode 100644 index 000000000..a924d75f9 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_modes_internal_test.go @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "context" + "errors" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/lockfile" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadLocklessComponents(t *testing.T) { + for _, testCase := range []struct { + name string + lines []string + missing bool + want map[string]bool + }{ + {name: "missing file", missing: true, want: map[string]bool{}}, + {name: "empty file", want: map[string]bool{}}, + { + name: "exact names with whitespace and duplicates", + lines: []string{" rust-asyncband \r", "", "\tcurl\t", "rust-asyncband", "Curl"}, + want: map[string]bool{"rust-asyncband": true, "curl": true, "Curl": true}, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + require.NoError(t, fileutils.WriteFile( + env.TestFS, "/project/subdir/lockless-components.txt", []byte("wrong-component"), fileperms.PublicFile, + )) + require.NoError(t, env.TestOSEnv.Chdir("/project/subdir")) + + if !testCase.missing { + require.NoError(t, fileutils.WriteFile( + env.TestFS, "/project/lockless-components.txt", + []byte(strings.Join(testCase.lines, "\n")), fileperms.PublicFile, + )) + } + + names, err := projectconfig.ReadLocklessComponents(env.TestFS, env.Env.ProjectDir()) + require.NoError(t, err) + assert.Equal(t, testCase.want, names) + }) + } + + t.Run("read errors are not treated as an empty list", func(t *testing.T) { + env := testutils.NewTestEnv(t) + require.NoError(t, fileutils.MkdirAll(env.TestFS, "/project/lockless-components.txt")) + + _, err := projectconfig.ReadLocklessComponents(env.TestFS, env.Env.ProjectDir()) + require.ErrorContains(t, err, "lockless-components.txt") + }) +} + +type renderModeObservingPreparer struct { + sources.SourcePreparer + observe func(components.Component) + err error +} + +func (p *renderModeObservingPreparer) PrepareSources( + _ context.Context, component components.Component, _ string, _ bool, +) error { + p.observe(component) + + return p.err +} + +func TestRenderComponentsSelectsLockModeFromFile(t *testing.T) { + for _, testCase := range []struct { + name string + withoutLockfile bool + }{ + {name: "default global mode"}, + {name: "lockless global mode", withoutLockfile: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + if testCase.withoutLockfile { + env = testutils.NewTestEnvWithoutLockfile(t) + env.Config.Project.LockDir = "/project/locks" + } + + for _, name := range []string{"curl", "rust-asyncband"} { + env.Config.Components[name] = projectconfig.ComponentConfig{ + Name: name, + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeUpstream, + UpstreamCommit: "configured-commit", + }, + } + } + + lock := lockfile.New() + lock.UpstreamCommit = "configured-commit" + env.WriteLock(t, "curl", lock) + require.NoError(t, fileutils.WriteFile( + env.TestFS, "/project/locks/rust-asyncband.lock", []byte("corrupt lock"), fileperms.PublicFile, + )) + require.NoError(t, fileutils.WriteFile( + env.TestFS, "/project/lockless-components.txt", []byte("rust-asyncband\n"), fileperms.PublicFile, + )) + names, err := projectconfig.ReadLocklessComponents(env.TestFS, env.Env.ProjectDir()) + require.NoError(t, err) + + env.Config.LocklessComponents = names + + var observedMutex sync.Mutex + + observed := make(map[string]*projectconfig.ComponentLockData) + observedErr := errors.New("stop after observing render mode") + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, _ opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return &renderModeObservingPreparer{ + observe: func(comp components.Component) { + observedMutex.Lock() + defer observedMutex.Unlock() + + observed[comp.GetName()] = comp.GetConfig().Locked + }, + err: observedErr, + }, nil + } + + t.Cleanup(func() { newSourcePreparer = original }) + + results, err := RenderComponents(env.Env, &RenderOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + OutputDir: "/rendered", + OutputDirExplicit: true, + }) + require.NoError(t, err) + require.Len(t, results, 2) + require.Len(t, observed, 2) + + if testCase.withoutLockfile { + assert.Nil(t, observed["curl"], "the global flag overrides the component list") + } else { + require.NotNil(t, observed["curl"]) + assert.Equal(t, "configured-commit", observed["curl"].UpstreamCommit) + } + + assert.Nil(t, observed["rust-asyncband"]) + assert.Equal(t, testCase.withoutLockfile, env.Env.WithoutLockfile()) + }) + } +} + +func TestRenderComponentsRequiresLocksForUnlistedComponents(t *testing.T) { + env := testutils.NewTestEnv(t) + env.AddUpstreamComponent(t, "curl") + env.Config.Components["rust-asyncband"] = projectconfig.ComponentConfig{ + Name: "rust-asyncband", + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + } + require.NoError(t, fileutils.WriteFile( + env.TestFS, filepath.Join(env.Env.ProjectDir(), "lockless-components.txt"), []byte("curl\n"), fileperms.PublicFile, + )) + env.Config.LocklessComponents = map[string]bool{"curl": true} + + _, err := RenderComponents(env.Env, &RenderOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + OutputDir: "/rendered", + OutputDirExplicit: true, + }) + require.ErrorContains(t, err, "lock file validation failed") +} diff --git a/internal/app/azldev/cmds/component/update.go b/internal/app/azldev/cmds/component/update.go index 9da1b4f75..34a7b2a03 100644 --- a/internal/app/azldev/cmds/component/update.go +++ b/internal/app/azldev/cmds/component/update.go @@ -59,6 +59,10 @@ For local components, this computes a content hash of the spec directory. Subsequent commands (render, build) use the locked state for deterministic, reproducible results. +Components listed in lockless-components.txt beside the top-level azldev.toml +are skipped, including with --bump. Use refresh-upstream-commit for them. +Their existing lock files are left untouched. + If a component fails to resolve, successful components are still written before the command returns an error. Failed components are left unchanged, and orphan pruning is skipped for that run. @@ -161,6 +165,14 @@ func UpdateComponents(env *azldev.Env, options *UpdateComponentOptions) ([]Updat return nil, fmt.Errorf("%w: --bump and --check-only are mutually exclusive", azldev.ErrInvalidUsage) } + if env.WithoutLockfile() { + return []UpdateResult{}, nil + } + + return updateLockedComponents(env, options) +} + +func updateLockedComponents(env *azldev.Env, options *UpdateComponentOptions) ([]UpdateResult, error) { resolver := components.NewResolver(env) // Suppress staleness warnings — we're about to refresh the locks ourselves, // so warning the user to "run component update" would be self-referential noise. @@ -184,6 +196,8 @@ func UpdateComponents(env *azldev.Env, options *UpdateComponentOptions) ([]Updat return nil, errors.New("no components matched the filter") } + lockedComps := componentsForLockMode(env, comps, false) + // Resolve upstream commits in parallel (no-op for empty list). store := env.LockStore() if store == nil { @@ -195,7 +209,7 @@ func UpdateComponents(env *azldev.Env, options *UpdateComponentOptions) ([]Updat // existing locks and should not delete entries for components that may // have been removed. if options.Bump { - results, bumpErr := bumpComponents(env, store, comps, options) + results, bumpErr := bumpComponents(env, store, lockedComps, options) if bumpErr != nil { return filterDisplayResults(results), bumpErr } @@ -205,7 +219,7 @@ func UpdateComponents(env *azldev.Env, options *UpdateComponentOptions) ([]Updat return filterDisplayResults(results), nil } - results := resolveSourceIdentitiesParallel(env, comps, store) + results := resolveSourceIdentitiesParallel(env, lockedComps, store) // Don't save if the context was cancelled (Ctrl+C). if env.Context().Err() != nil { diff --git a/internal/app/azldev/cmds/component/update_test.go b/internal/app/azldev/cmds/component/update_test.go index fea8742ab..dabcb759e 100644 --- a/internal/app/azldev/cmds/component/update_test.go +++ b/internal/app/azldev/cmds/component/update_test.go @@ -22,6 +22,41 @@ import ( const testLockDir = "/project/locks" +func TestUpdateComponents_SkipsLocklessAndPreservesTheirLocks(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + setupMockGit(env, "resolved-commit") + addUpstreamComponent(env, "curl") + addUpstreamComponent(env, "rust-asyncband") + + ignoredLock := []byte("corrupt unused lock") + require.NoError(t, fileutils.WriteFile(env.TestFS, testLockDir+"/rust-asyncband.lock", + ignoredLock, fileperms.PublicFile)) + + results, err := componentcmds.UpdateComponents(env.Env, &componentcmds.UpdateComponentOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + }) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "curl", results[0].Component) + + lock, err := env.Env.LockStore().Get("curl") + require.NoError(t, err) + assert.Equal(t, "resolved-commit", lock.UpstreamCommit) + + results, err = componentcmds.UpdateComponents(env.Env, &componentcmds.UpdateComponentOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + Bump: true, + }) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "curl", results[0].Component) + + unchanged, err := fileutils.ReadFile(env.TestFS, testLockDir+"/rust-asyncband.lock") + require.NoError(t, err) + assert.Equal(t, ignoredLock, unchanged) +} + func TestNewUpdateCmd(t *testing.T) { cmd := componentcmds.NewUpdateCmd() require.NotNil(t, cmd) diff --git a/internal/app/azldev/cmds/docs/agent.go b/internal/app/azldev/cmds/docs/agent.go index 0591e1e5f..793e1da49 100644 --- a/internal/app/azldev/cmds/docs/agent.go +++ b/internal/app/azldev/cmds/docs/agent.go @@ -89,8 +89,9 @@ MCP server. Directory paths in the emitted content (such as the lock, generated upstream-commit, and rendered-spec directories) are resolved from the loaded azldev.toml, falling back to azldev's built-in defaults when no configuration is -found. The emitted content also reflects the mode azldev runs in, so pass ---without-lockfile to describe the lock-file-free workflow. The bindings reflect +found. By default, the content includes both lock-file and lockless workflows +selected per component by lockless-components.txt. Pass --without-lockfile to +describe an entirely lockless project. The bindings reflect the project azldev runs in, so pair --output-dir with -C pointing at the target repository when scaffolding a different repo.`, Example: ` # Write agent files into the current repository diff --git a/internal/app/azldev/core/components/resolver.go b/internal/app/azldev/core/components/resolver.go index fad536a40..e4f0b2986 100644 --- a/internal/app/azldev/core/components/resolver.go +++ b/internal/app/azldev/core/components/resolver.go @@ -581,6 +581,12 @@ func (r *Resolver) createComponentFromConfig(componentConfig *projectconfig.Comp // goes into the separate Locked field, preserving the manifest/lock boundary: // Spec.UpstreamCommit = user intent, Locked.UpstreamCommit = resolved reality. func (r *Resolver) populateFromLock(config *projectconfig.ComponentConfig) { + if r.env.ComponentWithoutLockfile(config.Name) { + config.Locked = nil + + return + } + reader := r.env.LockReader() if reader == nil { return @@ -912,11 +918,17 @@ func (r *Resolver) validateLockFiles(resolved *ComponentSet, checkOrphans bool, // Build resolved config map from the component set. resolvedConfigs := make(map[string]projectconfig.ComponentConfig, resolved.Len()) + + var ignoredComponents []string + for _, comp := range resolved.Components() { resolvedConfigs[comp.GetName()] = *comp.GetConfig() + if r.env.ComponentWithoutLockfile(comp.GetName()) { + ignoredComponents = append(ignoredComponents, comp.GetName()) + } } - stale, orphans, err := reader.ValidateConsistency(resolvedConfigs, checkOrphans) + stale, orphans, err := reader.ValidateConsistency(resolvedConfigs, checkOrphans, ignoredComponents...) if err == nil { return nil } diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 4edeb30b6..ca6df85b3 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -114,7 +114,7 @@ func WithDirtyDetection() PreparerOption { // WithoutLockfileHistory returns a [PreparerOption] that derives synthetic history // from the component's generated upstream-commit TOML instead of its lock file. -// It selects the behavior of the global '--without-lockfile' flag; the lock reader +// It selects lockless component behavior; the lock reader // and release version passed to [WithGitRepo] are unused in that mode. func WithoutLockfileHistory() PreparerOption { return func(p *sourcePreparerImpl) { diff --git a/internal/app/azldev/env.go b/internal/app/azldev/env.go index 9b426033f..f567590bd 100644 --- a/internal/app/azldev/env.go +++ b/internal/app/azldev/env.go @@ -283,6 +283,12 @@ func (env *Env) WithoutLockfile() bool { return env.withoutLockfile } +// ComponentWithoutLockfile reports the selected component's mode. The global +// '--without-lockfile' flag overrides the project's per-component list. +func (env *Env) ComponentWithoutLockfile(name string) bool { + return env.withoutLockfile || (env.config != nil && env.config.LocklessComponents[name]) +} + // SetEventListener registers the event listener to be used in this environment. func (env *Env) SetEventListener(eventListener opctx.EventListener) { env.eventListener = eventListener diff --git a/internal/app/azldev/env_test.go b/internal/app/azldev/env_test.go index 7aa094960..2485fc638 100644 --- a/internal/app/azldev/env_test.go +++ b/internal/app/azldev/env_test.go @@ -10,6 +10,7 @@ import ( "log/slog" "os" "runtime" + "strconv" "sync" "testing" "time" @@ -178,6 +179,23 @@ func TestEnvWithCancel(t *testing.T) { assert.NoError(t, original.Err()) } +func TestEnvComponentWithoutLockfile(t *testing.T) { + for _, global := range []bool{false, true} { + t.Run(strconv.FormatBool(global), func(t *testing.T) { + env := testutils.NewTestEnv(t) + if global { + env = testutils.NewTestEnvWithoutLockfile(t) + } + + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + + assert.True(t, env.Env.ComponentWithoutLockfile("rust-asyncband")) + assert.Equal(t, global, env.Env.ComponentWithoutLockfile("curl")) + assert.Equal(t, global, env.Env.WithoutLockfile()) + }) + } +} + func TestFixSuggestions(t *testing.T) { t.Run("no suggestions does not panic", func(t *testing.T) { testEnv := testutils.NewTestEnv(t) diff --git a/internal/lockfile/lockfile.go b/internal/lockfile/lockfile.go index 34c55236a..988e9690f 100644 --- a/internal/lockfile/lockfile.go +++ b/internal/lockfile/lockfile.go @@ -214,12 +214,18 @@ func validateConsistency( lockDir string, components map[string]projectconfig.ComponentConfig, checkOrphans bool, + ignoredComponents ...string, ) (missingOrStale, orphans []string, err error) { + ignored := make(map[string]bool, len(ignoredComponents)) + for _, name := range ignoredComponents { + ignored[name] = true + } + // Check each component that may need a lock file. Only skip components // that are explicitly local — unspecified source type inherits upstream // from distro defaults, so those need validation too. for name, comp := range components { - if comp.Spec.SourceType == projectconfig.SpecSourceTypeLocal { + if ignored[name] || comp.Spec.SourceType == projectconfig.SpecSourceTypeLocal { continue } diff --git a/internal/lockfile/lockfile_test.go b/internal/lockfile/lockfile_test.go index 485b51340..f02a83873 100644 --- a/internal/lockfile/lockfile_test.go +++ b/internal/lockfile/lockfile_test.go @@ -456,6 +456,31 @@ func TestValidateUpstreamCommit(t *testing.T) { } func TestValidateConsistency(t *testing.T) { + t.Run("ignored components skip validation but preserve orphan detection", func(t *testing.T) { + memFS := afero.NewMemMapFs() + store := lockfile.NewStore(memFS, testLockDir) + components := map[string]projectconfig.ComponentConfig{ + "missing": {}, + "corrupt": {}, + "stale": {Spec: projectconfig.SpecSource{UpstreamCommit: "new-commit"}}, + "locked": {}, + } + + require.NoError(t, fileutils.WriteFile( + memFS, mustLockPath(t, "corrupt"), []byte("not valid TOML"), fileperms.PublicFile, + )) + + lock := lockfile.New() + lock.UpstreamCommit = testCommitHash + require.NoError(t, store.Save("stale", lock)) + require.NoError(t, store.Save("orphan", lock)) + + stale, orphans, err := store.ValidateConsistency(components, true, "missing", "corrupt", "stale") + require.Error(t, err) + assert.Equal(t, []string{"locked"}, stale) + assert.Equal(t, []string{"orphan"}, orphans) + }) + t.Run("all valid", func(t *testing.T) { memFS := afero.NewMemMapFs() store := lockfile.NewStore(memFS, testLockDir) diff --git a/internal/lockfile/show.go b/internal/lockfile/show.go index afeca998f..002584632 100644 --- a/internal/lockfile/show.go +++ b/internal/lockfile/show.go @@ -65,6 +65,7 @@ func ReadAllAtCommit( repo *gogit.Repository, commitHash string, lockRelDir string, + ignoredComponents ...string, ) (map[string]ComponentLock, error) { lockRelDir, err := cleanLockRelDir(lockRelDir) if err != nil { @@ -80,7 +81,7 @@ func ReadAllAtCommit( return make(map[string]ComponentLock), nil } - return parseLockEntries(lockSubtree, lockRelDir, commitHash) + return parseLockEntries(lockSubtree, lockRelDir, commitHash, ignoredComponents...) } // cleanLockRelDir normalizes and validates a repo-relative lock directory path. @@ -136,9 +137,15 @@ func resolveLockSubtree( // parseLockEntries reads all lock files from a resolved tree. func parseLockEntries( lockSubtree *object.Tree, lockRelDir, commitHash string, + ignoredComponents ...string, ) (map[string]ComponentLock, error) { locks := make(map[string]ComponentLock) + ignored := make(map[string]bool, len(ignoredComponents)) + for _, name := range ignoredComponents { + ignored[name] = true + } + for _, entry := range lockSubtree.Entries { if !entry.Mode.IsFile() || strings.HasPrefix(entry.Name, ".") || @@ -147,6 +154,10 @@ func parseLockEntries( } name := strings.TrimSuffix(entry.Name, lockFileExtension) + if ignored[name] { + continue + } + entryPath := gopath.Join(lockRelDir, entry.Name) file, fileErr := lockSubtree.File(entry.Name) diff --git a/internal/lockfile/show_test.go b/internal/lockfile/show_test.go index 3ede47dd7..1ca7997b7 100644 --- a/internal/lockfile/show_test.go +++ b/internal/lockfile/show_test.go @@ -152,6 +152,19 @@ func TestShowAtCommit(t *testing.T) { func TestReadAllAtCommit(t *testing.T) { const lockDir = "locks" + t.Run("ignored corrupt locks are not read", func(t *testing.T) { + testFS := memfs.New() + repo, err := gogit.Init(memory.NewStorage(), testFS) + require.NoError(t, err) + commitFile(t, repo, testFS, "locks/curl.lock", validLockTOML, "add curl lock") + hash := commitFile(t, repo, testFS, "locks/rust-asyncband.lock", "invalid TOML", "add unused lock") + + locks, err := lockfile.ReadAllAtCommit(repo, hash, lockDir, "rust-asyncband") + require.NoError(t, err) + require.Len(t, locks, 1) + assert.Contains(t, locks, "curl") + }) + t.Run("reads all locks", func(t *testing.T) { testFS := memfs.New() repo, err := gogit.Init(memory.NewStorage(), testFS) diff --git a/internal/lockfile/store.go b/internal/lockfile/store.go index a031863ba..efa25fdb0 100644 --- a/internal/lockfile/store.go +++ b/internal/lockfile/store.go @@ -26,10 +26,12 @@ type LockReader interface { LockDir() string // ValidateConsistency checks lock files against the resolved component // configs. Returns sorted lists of components with missing/stale locks - // and orphan component names. + // and orphan component names. ignoredComponents are exempt from lock + // validation but still count as configured components for orphan detection. ValidateConsistency( components map[string]projectconfig.ComponentConfig, checkOrphans bool, + ignoredComponents ...string, ) (missingOrStale, orphans []string, err error) } @@ -210,8 +212,9 @@ func (s *Store) Remove(componentName string) error { func (s *Store) ValidateConsistency( components map[string]projectconfig.ComponentConfig, checkOrphans bool, + ignoredComponents ...string, ) (missingOrStale, orphans []string, err error) { - return validateConsistency(s.fs, s.lockDir, components, checkOrphans) + return validateConsistency(s.fs, s.lockDir, components, checkOrphans, ignoredComponents...) } // FindOrphanLockFiles returns component names that have lock files but no diff --git a/internal/projectconfig/config.go b/internal/projectconfig/config.go index f3d320bef..edabfa16b 100644 --- a/internal/projectconfig/config.go +++ b/internal/projectconfig/config.go @@ -18,9 +18,8 @@ import ( // for cleaning it up -- but not until after it is done using the loaded configuration. The loaded // configuration may implicitly depend on the contents of the temporary directory. // -// When withoutLockfile is set, component definitions merge with override semantics, -// are validated only once the whole project is assembled, and the project's lock -// directory is left unset. +// Lockless component definitions merge with override semantics and are validated +// once the whole project is assembled. The global mode overrides the component list. func LoadProjectConfig( fs opctx.FS, osEnv opctx.OSEnv, @@ -37,6 +36,11 @@ func LoadProjectConfig( return "", nil, fmt.Errorf("failed to find project root and config file:\n%w", err) } + locklessComponents, err := ReadLocklessComponents(fs, projectDir) + if err != nil { + return "", nil, err + } + configFilePaths := []string{} // Unless we were explicitly requested not to do so, copy our embedded default config files to a temporary @@ -87,6 +91,7 @@ func LoadProjectConfig( options := loadOptions{ permissiveConfigParsing: permissiveConfigParsing, withoutLockfile: withoutLockfile, + locklessComponents: locklessComponents, } config, err = loadAndResolveProjectConfig(fs, options, configFilePaths...) @@ -100,12 +105,5 @@ func LoadProjectConfig( // Apply project-relative defaults for any unset path fields. config.Project.ApplyProjectDefaults(projectDir) - // Lock-file-free mode never reads or writes lock files, so the lock directory - // is left unset. A project may still declare 'lock-dir' for compatibility with - // azldev's default mode; the value is simply ignored here. - if withoutLockfile { - config.Project.LockDir = "" - } - return projectDir, config, nil } diff --git a/internal/projectconfig/config_test.go b/internal/projectconfig/config_test.go index 5912b2c47..f807b8513 100644 --- a/internal/projectconfig/config_test.go +++ b/internal/projectconfig/config_test.go @@ -41,6 +41,86 @@ func writeProjectConfig(t *testing.T, ctx *testctx.TestCtx, contents string) { []byte(contents), fileperms.PublicFile)) } +func TestLoadProjectConfig_PerComponentLockModes(t *testing.T) { + ctx := newTestCtxWithXDGConfigHome() + writeProjectConfig(t, ctx, ` +includes = ["pins.toml", "components.toml"] + +[components.curl.spec] +type = "upstream" +upstream-commit = "aaa1111" + +[components.rust-asyncband.spec] +type = "upstream" +upstream-commit = "bbb1111" +`) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(testProjectDir, "pins.toml"), []byte(` +[components.curl.spec] +upstream-commit = "aaa2222" +[components.rust-asyncband.spec] +upstream-commit = "bbb2222" +`), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(testProjectDir, "components.toml"), []byte(` +[components.curl.release] +calculation = "static" +[components.rust-asyncband.release] +calculation = "static" +`), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(testProjectDir, "lockless-components.txt"), + []byte("rust-asyncband\n"), fileperms.PublicFile)) + + for _, global := range []bool{false, true} { + _, config, err := projectconfig.LoadProjectConfig( + ctx.FS(), ctx.OSEnv(), testProjectDir, true, "/tmp", nil, false, global, + ) + require.NoError(t, err) + assert.Equal(t, map[string]bool{"rust-asyncband": true}, config.LocklessComponents) + rustComponent := config.Components["rust-asyncband"] + assert.Equal(t, "bbb2222", rustComponent.Spec.UpstreamCommit) + assert.Equal(t, filepath.Join(testProjectDir, "pins.toml"), + rustComponent.UpstreamCommitConfigFile().SourcePath()) + + curlComponent := config.Components["curl"] + assert.Equal(t, "aaa2222", curlComponent.Spec.UpstreamCommit) + + expectedSource := "components.toml" + if global { + expectedSource = "pins.toml" + } + + assert.Equal(t, filepath.Join(testProjectDir, expectedSource), + curlComponent.UpstreamCommitConfigFile().SourcePath()) + } +} + +func TestLoadProjectConfig_LockModesValidation(t *testing.T) { + for _, listed := range []string{"rust-asyncband", "curl"} { + t.Run(listed, func(t *testing.T) { + ctx := newTestCtxWithXDGConfigHome() + writeProjectConfig(t, ctx, ` +includes = ["later.toml"] +[components.rust-asyncband.release] +calculation = "invalid" +`) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(testProjectDir, "later.toml"), []byte(` +[components.rust-asyncband.release] +calculation = "static" +`), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(testProjectDir, "lockless-components.txt"), + []byte(listed+"\n"), fileperms.PublicFile)) + + _, _, err := projectconfig.LoadProjectConfig( + ctx.FS(), ctx.OSEnv(), testProjectDir, true, "/tmp", nil, false, false, + ) + if listed == "rust-asyncband" { + require.NoError(t, err) + } else { + require.Error(t, err, "unlisted component definitions must still be validated per file") + } + }) + } +} + // TestLoadProjectConfig_NoUserConfig verifies that loading succeeds (and produces a // project-derived value) when no user-level config file is present. func TestLoadProjectConfig_NoUserConfig(t *testing.T) { @@ -173,9 +253,8 @@ description = "`+testUserDesc+`" assert.Equal(t, testUserDesc, config.Project.Description) } -// TestLoadProjectConfig_LockDirByMode verifies that the project's lock directory is -// defaulted in azldev's default mode and left unset in lock-file-free mode, where an -// explicitly configured 'lock-dir' is accepted but ignored. +// TestLoadProjectConfig_LockDirByMode verifies that the project's lock directory +// remains available in either global mode for per-component rendering. func TestLoadProjectConfig_LockDirByMode(t *testing.T) { testCases := []struct { name string @@ -183,7 +262,7 @@ func TestLoadProjectConfig_LockDirByMode(t *testing.T) { expected string }{ {name: "default mode", withoutLockfile: false, expected: filepath.Join(testProjectDir, "legacy-locks")}, - {name: "lock-file-free mode", withoutLockfile: true, expected: ""}, + {name: "lock-file-free mode", withoutLockfile: true, expected: filepath.Join(testProjectDir, "legacy-locks")}, } for _, testCase := range testCases { diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index ee6f07c73..c19beae03 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -127,13 +127,17 @@ func (f ConfigFile) Validate() error { return nil } -// validateNonComponentFields validates every field except the component -// definitions. Lock-file-free mode merges component definitions across config -// files with override semantics, so a single file may legitimately be -// incomplete; components are validated once the whole project is assembled. -func (f ConfigFile) validateNonComponentFields() error { +// validateForLockModes defers validation of partial lockless definitions until +// the project is assembled, while validating locked definitions immediately. +func (f ConfigFile) validateForLockModes(options loadOptions) error { components := f.Components - f.Components = nil + f.Components = make(map[string]ComponentConfig) + + for name, component := range components { + if !options.componentWithoutLockfile(name) { + f.Components[name] = component + } + } if err := f.Validate(); err != nil { return err @@ -142,6 +146,10 @@ func (f ConfigFile) validateNonComponentFields() error { // Source-file entries are atomic slice elements rather than partial nested // definitions, so validate them before their script paths become absolute. for componentName, component := range components { + if !options.componentWithoutLockfile(componentName) { + continue + } + if err := validateSourceFiles(component.SourceFiles, componentName, false); err != nil { return err } diff --git a/internal/projectconfig/loader.go b/internal/projectconfig/loader.go index 79af6ca08..b0131a2d2 100644 --- a/internal/projectconfig/loader.go +++ b/internal/projectconfig/loader.go @@ -38,7 +38,12 @@ type loadOptions struct { // withoutLockfile selects lock-file-free mode, where component definitions are // merged with override semantics and validated only once fully assembled. - withoutLockfile bool + withoutLockfile bool + locklessComponents map[string]bool +} + +func (options loadOptions) componentWithoutLockfile(name string) bool { + return options.withoutLockfile || options.locklessComponents[name] } // Loads and resolves the project configuration files located at the given path. Referenced include files @@ -48,6 +53,7 @@ func loadAndResolveProjectConfig( fs opctx.FS, options loadOptions, configFilePaths ...string, ) (*ProjectConfig, error) { defaultConfig := NewProjectConfig() + defaultConfig.LocklessComponents = options.locklessComponents resolvedCfg := &defaultConfig for _, configFilePath := range configFilePaths { @@ -244,7 +250,23 @@ func mergeComponents(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile, options return mergeComponentsWithOverride(resolvedCfg, loadedCfg) } - return mergeComponentsAdditively(resolvedCfg, loadedCfg) + locked, lockless := *loadedCfg, *loadedCfg + locked.Components = make(map[string]ComponentConfig) + + lockless.Components = make(map[string]ComponentConfig) + for name, component := range loadedCfg.Components { + if options.componentWithoutLockfile(name) { + lockless.Components[name] = component + } else { + locked.Components[name] = component + } + } + + if err := mergeComponentsAdditively(resolvedCfg, &locked); err != nil { + return err + } + + return mergeComponentsWithOverride(resolvedCfg, &lockless) } // mergeComponentsAdditively merges component definitions from a loaded config file @@ -548,16 +570,10 @@ func loadProjectConfigFile( cfg.sourcePath = absFilePath cfg.dir = filepath.Dir(absFilePath) - // Make sure that the read data is internally consistent. In lock-file-free - // mode, component definitions support override merging across files, so they - // are validated only after the complete project configuration is assembled. + // Lockless component definitions support override merging across files, so + // validate them only after the complete configuration is assembled. // Syntax and unknown fields were already checked by the TOML decoder. - if options.withoutLockfile { - err = cfg.validateNonComponentFields() - } else { - err = cfg.Validate() - } - + err = cfg.validateForLockModes(options) if err != nil { return nil, err } diff --git a/internal/projectconfig/lockmodes.go b/internal/projectconfig/lockmodes.go new file mode 100644 index 000000000..445eca702 --- /dev/null +++ b/internal/projectconfig/lockmodes.go @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +// ReadLocklessComponents reads the exact component names listed in +// 'lockless-components.txt' beside the project's top-level 'azldev.toml'. +func ReadLocklessComponents(fs opctx.FS, projectDir string) (map[string]bool, error) { + names := make(map[string]bool) + if projectDir == "" { + return names, nil + } + + path := filepath.Join(projectDir, "lockless-components.txt") + + info, err := fs.Stat(path) + if errors.Is(err, os.ErrNotExist) { + return names, nil + } + + if err != nil { + return nil, fmt.Errorf("reading %#q:\n%w", path, err) + } + + if info.IsDir() { + return nil, fmt.Errorf("expected a file at %#q, found a directory", path) + } + + content, err := fileutils.ReadFile(fs, path) + if err != nil { + return nil, fmt.Errorf("reading %#q:\n%w", path, err) + } + + for line := range strings.SplitSeq(string(content), "\n") { + if name := strings.TrimSpace(line); name != "" { + names[name] = true + } + } + + return names, nil +} diff --git a/internal/projectconfig/project.go b/internal/projectconfig/project.go index 605066b80..61328c095 100644 --- a/internal/projectconfig/project.go +++ b/internal/projectconfig/project.go @@ -61,6 +61,8 @@ type ProjectConfig struct { RootConfigFilePath string `toml:"-" json:"-"` // Map from component names to groups they belong to; not serialized. GroupsByComponent map[string][]string `toml:"-" json:"-"` + // LocklessComponents is the project-root component selection; not serialized. + LocklessComponents map[string]bool `toml:"-" json:"-"` } // Constructs a default (empty) project configuration. @@ -86,23 +88,16 @@ func (cfg *ProjectConfig) Validate() error { return cfg.validate(false) } -// validate checks the assembled project configuration. In lock-file-free mode the -// component definitions are validated here rather than per config file, because -// override merging lets a single file hold a partial definition. +// validate checks the assembled project configuration. Lockless component +// definitions are validated here because a single file may be partial. func (cfg *ProjectConfig) validate(withoutLockfile bool) error { err := projectConfigValidator.Struct(cfg) if err != nil { return fmt.Errorf("config error:\n%w", err) } - if withoutLockfile { - if err := validateComponentStructs(cfg.Components); err != nil { - return err - } - - if err := validateComponentConfigs(cfg.Components, true); err != nil { - return err - } + if err := cfg.validateLocklessComponents(withoutLockfile); err != nil { + return err } if err := validateComponentGroupMembership(cfg.ComponentGroups, cfg.Components); err != nil { @@ -147,6 +142,22 @@ func (cfg *ProjectConfig) validate(withoutLockfile bool) error { return nil } +func (cfg *ProjectConfig) validateLocklessComponents(withoutLockfile bool) error { + lockless := make(map[string]ComponentConfig) + + for name, component := range cfg.Components { + if withoutLockfile || cfg.LocklessComponents[name] { + lockless[name] = component + } + } + + if err := validateComponentStructs(lockless); err != nil { + return err + } + + return validateComponentConfigs(lockless, true) +} + // validateImages runs the image-scoped validations: capabilities, architectures, // and SKU groups. func validateImages(cfg *ProjectConfig) error { diff --git a/scenario/without_lockfile_test.go b/scenario/without_lockfile_test.go index 23ccd4cbc..53df92409 100644 --- a/scenario/without_lockfile_test.go +++ b/scenario/without_lockfile_test.go @@ -22,9 +22,8 @@ import ( "github.com/stretchr/testify/require" ) -// These tests cover the opt-in lock-file-free mode selected by the global -// '--without-lockfile' flag, and assert that omitting the flag keeps azldev's -// default lock-file behavior. +// These tests cover the global lockless override and per-component selection +// through 'lockless-components.txt'. // TestWithoutLockfile_GlobalFlagIsDocumented verifies that the preview flag is part // of the CLI surface and defaults to off. @@ -42,8 +41,7 @@ func TestWithoutLockfile_GlobalFlagIsDocumented(t *testing.T) { } // TestWithoutLockfile_ComponentCommandsByMode verifies that the component command -// set matches the selected mode: the lock-file commands by default, and the -// refresh-upstream-commit command when the preview flag is passed. +// set exposes both workflows by default and the lockless workflow with the flag. func TestWithoutLockfile_ComponentCommandsByMode(t *testing.T) { t.Parallel() @@ -60,14 +58,12 @@ func TestWithoutLockfile_ComponentCommandsByMode(t *testing.T) { { name: "default mode", args: []string{"component", "--help"}, - expected: []string{"update", "history", "query"}, - notExpected: []string{"refresh-upstream-commit"}, + expected: []string{"update", "history", "query", "refresh-upstream-commit"}, }, { name: "explicitly disabled", args: []string{"--without-lockfile=false", "component", "--help"}, - expected: []string{"update", "history", "query"}, - notExpected: []string{"refresh-upstream-commit"}, + expected: []string{"update", "history", "query", "refresh-upstream-commit"}, }, { name: "lock-file-free mode", @@ -150,7 +146,7 @@ func TestWithoutLockfile_AgentSkillsByMode(t *testing.T) { require.NoError(t, err) require.Zero(t, defaultResults.ExitCode) assert.Contains(t, defaultResults.Stdout, "azldev-update-component") - assert.NotContains(t, defaultResults.Stdout, "azldev-refresh-upstream-commit") + assert.Contains(t, defaultResults.Stdout, "azldev-refresh-upstream-commit") previewResults, err := cmdtest.NewScenarioTest( "--without-lockfile", "docs", "agent", "show", @@ -179,6 +175,22 @@ func TestWithoutLockfile_ComponentChanged(t *testing.T) { t.Skip("skipping long test") } + testLocklessComponentChanged(t, true) +} + +func TestLocklessComponents_ComponentChanged(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping long test") + } + + testLocklessComponentChanged(t, false) +} + +func testLocklessComponentChanged(t *testing.T, global bool) { + t.Helper() + azldevBin, err := testhelpers.FindTestBinary() require.NoError(t, err) @@ -216,6 +228,10 @@ func TestWithoutLockfile_ComponentChanged(t *testing.T) { project.Serialize(t, projectDir) patchProjectForLocal(t, projectDir) + if !global { + require.NoError(t, os.WriteFile(filepath.Join(projectDir, "lockless-components.txt"), + []byte("curl\n"), fileperms.PublicFile)) + } gitInDir(t, projectDir, "init") gitInDir(t, projectDir, "config", "user.email", "test@test.com") @@ -238,10 +254,14 @@ func TestWithoutLockfile_ComponentChanged(t *testing.T) { gitInDir(t, projectDir, "add", filepath.Join("specs", "curl", "curl.spec")) gitInDir(t, projectDir, "-c", "commit.gpgsign=false", "commit", "-m", "change curl") - cmd := exec.CommandContext(t.Context(), - azldevBin, "--without-lockfile", "-C", projectDir, "--no-default-config", + args := []string{ + "-C", projectDir, "--no-default-config", "component", "changed", "--from", fromRef, "-a", "--include-unchanged", "-q", "-O", "json", - ) + } + if global { + args = append([]string{"--without-lockfile"}, args...) + } + cmd := exec.CommandContext(t.Context(), azldevBin, args...) out, err := cmd.CombinedOutput() require.NoError(t, err, "azldev failed: %s", string(out)) From d84200e079f5b87b3c683962929ca8042fe80c9c Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 30 Oct 2026 02:25:51 +0000 Subject: [PATCH 02/70] refactor(cli): remove global without-lockfile flag Make lockless-components.txt the sole CLI selector for locked and lockless component workflows. Remove the global flag, its preparsing and App state, mode-specific command options, and hidden legacy no-op commands. Always expose both update and refresh-upstream-commit and emit the shared mixed-workflow agent catalog. Retain internal lockless overrides used for historical comparisons and test fixtures. Update regression tests, scenario coverage, user documentation, emitted agent guidance, and generated CLI references. Validation: affected unit packages, mage fix all, mage docs, mage check all, and CLI rejection/help/catalog smoke checks pass. mage all encounters nine existing bumpspec failures. mage scenarioUpdate remains blocked by enforcing SELinux and an unavailable Docker provider. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3497f2b2-c7fb-456e-a58f-6889b6ec67e2 --- .../instructions/agent-skills.instructions.md | 17 +- docs/user/README.md | 2 +- docs/user/explanation/config-system.md | 2 +- docs/user/how-to/preview-without-lockfile.md | 28 ++-- docs/user/reference/cli/azldev.md | 1 - docs/user/reference/cli/azldev_advanced.md | 1 - .../cli/azldev_advanced_download-sources.md | 1 - .../user/reference/cli/azldev_advanced_mcp.md | 1 - .../reference/cli/azldev_advanced_mock.md | 1 - .../cli/azldev_advanced_mock_build-rpms.md | 1 - .../cli/azldev_advanced_mock_shell.md | 1 - .../reference/cli/azldev_advanced_wget.md | 1 - docs/user/reference/cli/azldev_completion.md | 1 - .../reference/cli/azldev_completion_bash.md | 1 - .../reference/cli/azldev_completion_fish.md | 1 - .../cli/azldev_completion_powershell.md | 1 - .../reference/cli/azldev_completion_zsh.md | 1 - docs/user/reference/cli/azldev_component.md | 1 - .../reference/cli/azldev_component_add.md | 1 - .../reference/cli/azldev_component_build.md | 1 - .../reference/cli/azldev_component_changed.md | 4 +- .../cli/azldev_component_diff-sources.md | 1 - .../reference/cli/azldev_component_history.md | 1 - .../reference/cli/azldev_component_list.md | 1 - .../cli/azldev_component_metadata.md | 1 - .../cli/azldev_component_prepare-sources.md | 1 - .../reference/cli/azldev_component_query.md | 1 - ...zldev_component_refresh-upstream-commit.md | 6 +- .../reference/cli/azldev_component_render.md | 3 +- .../reference/cli/azldev_component_test.md | 1 - .../reference/cli/azldev_component_update.md | 1 - docs/user/reference/cli/azldev_config.md | 1 - docs/user/reference/cli/azldev_config_dump.md | 1 - .../cli/azldev_config_generate-schema.md | 1 - docs/user/reference/cli/azldev_docs.md | 1 - docs/user/reference/cli/azldev_docs_agent.md | 1 - .../cli/azldev_docs_agent_install.md | 4 +- .../reference/cli/azldev_docs_agent_show.md | 1 - .../reference/cli/azldev_docs_markdown.md | 1 - docs/user/reference/cli/azldev_image.md | 1 - docs/user/reference/cli/azldev_image_boot.md | 1 - docs/user/reference/cli/azldev_image_build.md | 1 - .../reference/cli/azldev_image_customize.md | 1 - .../cli/azldev_image_inject-files.md | 1 - docs/user/reference/cli/azldev_image_list.md | 1 - docs/user/reference/cli/azldev_image_test.md | 1 - docs/user/reference/cli/azldev_package.md | 1 - .../user/reference/cli/azldev_package_list.md | 1 - docs/user/reference/cli/azldev_project.md | 1 - .../user/reference/cli/azldev_project_init.md | 1 - docs/user/reference/cli/azldev_project_new.md | 1 - docs/user/reference/cli/azldev_repo.md | 1 - .../user/reference/cli/azldev_repo_compare.md | 1 - docs/user/reference/cli/azldev_repo_query.md | 1 - docs/user/reference/cli/azldev_version.md | 1 - docs/user/reference/config/components.md | 3 +- docs/user/reference/config/config-file.md | 2 +- internal/app/azldev/agentskill/agentskill.go | 4 +- .../azldev/agentskill/content/azldev.md.tmpl | 3 +- .../agentskill/content/comp-toml.md.tmpl | 2 +- .../content/refresh-upstream-commit.md.tmpl | 5 +- .../content/update-component.md.tmpl | 1 - .../content/withoutlockfile/comp-toml.md.tmpl | 2 +- internal/app/azldev/agentskill/doc.go | 5 +- internal/app/azldev/app.go | 63 +------- internal/app/azldev/app_test.go | 61 ++----- internal/app/azldev/cmds/component/build.go | 8 +- internal/app/azldev/cmds/component/changed.go | 45 ++---- .../app/azldev/cmds/component/changed_test.go | 12 +- .../app/azldev/cmds/component/cmdoptions.go | 50 +----- .../app/azldev/cmds/component/component.go | 15 +- .../azldev/cmds/component/component_test.go | 47 ++---- .../app/azldev/cmds/component/diffsources.go | 8 +- internal/app/azldev/cmds/component/history.go | 2 +- internal/app/azldev/cmds/component/legacy.go | 41 ----- .../cmds/component/legacy_internal_test.go | 67 -------- internal/app/azldev/cmds/component/list.go | 13 +- .../azldev/cmds/component/preparesources.go | 8 +- internal/app/azldev/cmds/component/query.go | 2 +- .../cmds/component/refreshupstreamcommit.go | 7 +- .../component/refreshupstreamcommit_test.go | 27 +--- internal/app/azldev/cmds/component/render.go | 6 +- .../component/render_modes_internal_test.go | 6 +- internal/app/azldev/cmds/component/update.go | 2 +- internal/app/azldev/cmds/docs/agent.go | 19 +-- internal/app/azldev/core/testutils/testenv.go | 2 +- internal/app/azldev/env.go | 8 +- pkg/app/azldev_cli/azldev.go | 8 +- scenario/without_lockfile_test.go | 149 ++---------------- 89 files changed, 145 insertions(+), 671 deletions(-) delete mode 100644 internal/app/azldev/cmds/component/legacy.go delete mode 100644 internal/app/azldev/cmds/component/legacy_internal_test.go diff --git a/.github/instructions/agent-skills.instructions.md b/.github/instructions/agent-skills.instructions.md index 21ef77485..a68b3aff3 100644 --- a/.github/instructions/agent-skills.instructions.md +++ b/.github/instructions/agent-skills.instructions.md @@ -68,10 +68,9 @@ Before shipping any skill/instruction: ## Mode-specific content -azldev has two modes, and the emitted content follows the one it runs in. A `Catalog` -(`NewCatalog(withoutLockfile)`) resolves the skills, instruction wrappers, and templates; -the package-level `Skills`, `Instructions`, `FindSkill`, `SkillDocument`, and `Files` -helpers are the default (lock-file) mode. +The CLI always emits the default catalog, covering both workflows selected by +`lockless-components.txt`. `NewCatalog(true)` retains an internal all-lockless +variant; the package-level helpers use the default mixed-workflow catalog. - **Shared by default.** Registry entries and templates under `content/` describe the default mode and are used by both, so most edits need nothing extra. @@ -82,11 +81,11 @@ helpers are the default (lock-file) mode. same-named default template for that mode; add one only when the document's content actually differs. -When you add or edit a skill, check whether its content names a command that exists in -only one mode (for example `comp update` versus `comp refresh-upstream-commit`) and, if -so, provide the mode-specific variant. Verify both with -`./out/bin/azldev docs agent show --skill ` and -`./out/bin/azldev --without-lockfile docs agent show --skill `. +When editing a skill, describe which components each command processes: +`comp update` handles locked components; `comp refresh-upstream-commit` handles +listed lockless components. Verify CLI output with +`./out/bin/azldev docs agent show --skill ` and cover catalog variants in +unit tests. There is no global CLI mode selector. ## Config-resolved bindings diff --git a/docs/user/README.md b/docs/user/README.md index e94e49b67..fe38aa55d 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -7,7 +7,7 @@ - [Build a Component](./how-to/build-component.md) — build RPMs from component definitions - [Build an Image](./how-to/build-image.md) — build and boot Azure Linux images - [Set Up AI Coding Agents](./how-to/set-up-ai-agents.md) — emit agent skill and instruction files -- [Preview the Lock-File-Free Mode](./how-to/preview-without-lockfile.md) — select lockless components or use `--without-lockfile` +- [Preview the Lock-File-Free Mode](./how-to/preview-without-lockfile.md) — select lockless components ## Explanation diff --git a/docs/user/explanation/config-system.md b/docs/user/explanation/config-system.md index abfe20462..b87f2de50 100644 --- a/docs/user/explanation/config-system.md +++ b/docs/user/explanation/config-system.md @@ -99,7 +99,7 @@ Component definitions are merged additively. If the same component name (e.g., ` > **Note:** Slice fields (like `overlays`) are **appended**, not replaced, following the same merge behavior used by component configuration inheritance. -> **Preview:** Components listed in `lockless-components.txt` beside the top-level `azldev.toml` merge with override semantics instead, and their validation runs only after every included file has been merged, so an individual file may hold a partial component definition. Unlisted components retain additive merging and per-file validation. The global `--without-lockfile` flag applies the lockless behavior to every component. See [Preview the Lock-File-Free Mode](../how-to/preview-without-lockfile.md). +> **Preview:** Components listed in `lockless-components.txt` beside the top-level `azldev.toml` merge with override semantics instead, and their validation runs only after every included file has been merged, so an individual file may hold a partial component definition. Unlisted components retain additive merging and per-file validation. See [Preview the Lock-File-Free Mode](../how-to/preview-without-lockfile.md). ### Component Groups and Images diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 14f2fbc23..fd46ea5a0 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -2,8 +2,8 @@ The **preview** lockless workflow tracks a component's resolved upstream commit in generated component TOML instead of per-component lock files. Select it for -individual components using `lockless-components.txt`, or for every component -using the global `--without-lockfile` flag. +individual components using `lockless-components.txt`. To use it for the whole +project, list every component. The file applies to every command, not just rendering. Nothing in the preview workflow is stable yet; both the command surface and generated layout may change. @@ -16,10 +16,6 @@ azldev component render -p curl azldev component refresh-upstream-commit -p curl ``` -The flag overrides the file: when set, every component is lockless, including -unlisted components. With the flag absent or set to `false`, the file selects -each component's mode. - ## Select Lock Mode Per Component Create `lockless-components.txt` beside the top-level `azldev.toml`. List one @@ -40,8 +36,8 @@ A missing or empty file means all components use lock files. Blank lines and surrounding whitespace are ignored; names are not glob patterns. A file read error stops configuration loading. -Both `component update` and `component refresh-upstream-commit` are available -without the flag. Update (including `--bump`) skips listed components and leaves +Both `component update` and `component refresh-upstream-commit` are available. +Update (including `--bump`) skips listed components and leaves their locks untouched; refresh skips unlisted components and preserves their generated TOML files. All-component scans still prune files for components that no longer exist, not files belonging to components using the other mode. @@ -60,12 +56,7 @@ the current checkout's list determines which comparison to use. | Lock consistency checks | On, with `--skip-lock-validation` to opt out | Skipped | | `component changed` | Compares stored input fingerprints | Compares project configuration resolved at each ref | | Synthetic dist-git history | Derived from lock-file fingerprint changes | Derived from generated upstream-commit TOML changes | -| Agent skills and MCP tools | Both workflows are available for mixed projects | The global flag selects the entirely lockless catalog | - -`component update`, `component history`, and `component query` remain registered -in preview mode as hidden no-ops so that existing scripts report clearly that the -commands do nothing, rather than failing with "unknown command". Without the -global flag, they process only locked components. +| Agent skills and MCP tools | Both workflows are available | Both workflows are available | ## Configure the Project @@ -132,15 +123,14 @@ fields do not mark a component as changed. ## Emit Agent Files for the Preview Mode -`azldev docs agent install` includes both workflows for mixed projects. Pass the -flag to emit content for an entirely lockless project: +`azldev docs agent install` includes both workflows, selected per component by +the project's list: ```bash -azldev --without-lockfile docs agent install +azldev docs agent install ``` ## Reference Documentation The generated CLI reference under [reference/cli/](../reference/cli/azldev.md) -documents azldev's default mode. Use `azldev --without-lockfile --help` -to see the preview mode's command surface and help text. +documents both workflows. Use `azldev --help` for command syntax. diff --git a/docs/user/reference/cli/azldev.md b/docs/user/reference/cli/azldev.md index c5de33703..84dd91958 100644 --- a/docs/user/reference/cli/azldev.md +++ b/docs/user/reference/cli/azldev.md @@ -30,7 +30,6 @@ lives), or use -C to point to one. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced.md b/docs/user/reference/cli/azldev_advanced.md index f76116d52..2903a74ef 100644 --- a/docs/user/reference/cli/azldev_advanced.md +++ b/docs/user/reference/cli/azldev_advanced.md @@ -33,7 +33,6 @@ output but fully supported. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_download-sources.md b/docs/user/reference/cli/azldev_advanced_download-sources.md index 7758ecd08..cabe219da 100644 --- a/docs/user/reference/cli/azldev_advanced_download-sources.md +++ b/docs/user/reference/cli/azldev_advanced_download-sources.md @@ -71,7 +71,6 @@ azldev advanced download-sources [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mcp.md b/docs/user/reference/cli/azldev_advanced_mcp.md index 1f62c5e0c..abf04adaa 100644 --- a/docs/user/reference/cli/azldev_advanced_mcp.md +++ b/docs/user/reference/cli/azldev_advanced_mcp.md @@ -44,7 +44,6 @@ azldev advanced mcp [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mock.md b/docs/user/reference/cli/azldev_advanced_mock.md index 984278eea..849e14746 100644 --- a/docs/user/reference/cli/azldev_advanced_mock.md +++ b/docs/user/reference/cli/azldev_advanced_mock.md @@ -32,7 +32,6 @@ starting interactive shell sessions in mock chroot environments. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md b/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md index 18e1be1a3..9068693c3 100644 --- a/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md +++ b/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md @@ -51,7 +51,6 @@ azldev advanced mock build-rpms [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mock_shell.md b/docs/user/reference/cli/azldev_advanced_mock_shell.md index 54d6ebc76..7c5bf9e28 100644 --- a/docs/user/reference/cli/azldev_advanced_mock_shell.md +++ b/docs/user/reference/cli/azldev_advanced_mock_shell.md @@ -56,7 +56,6 @@ azldev advanced mock shell [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_wget.md b/docs/user/reference/cli/azldev_advanced_wget.md index aa23ba04a..cd5623b7a 100644 --- a/docs/user/reference/cli/azldev_advanced_wget.md +++ b/docs/user/reference/cli/azldev_advanced_wget.md @@ -45,7 +45,6 @@ azldev advanced wget URI [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion.md b/docs/user/reference/cli/azldev_completion.md index 109d7858e..6ec224917 100644 --- a/docs/user/reference/cli/azldev_completion.md +++ b/docs/user/reference/cli/azldev_completion.md @@ -31,7 +31,6 @@ See each sub-command's help for details on how to use the generated script. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_bash.md b/docs/user/reference/cli/azldev_completion_bash.md index f39580feb..74fe45a56 100644 --- a/docs/user/reference/cli/azldev_completion_bash.md +++ b/docs/user/reference/cli/azldev_completion_bash.md @@ -54,7 +54,6 @@ azldev completion bash -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_fish.md b/docs/user/reference/cli/azldev_completion_fish.md index 42eaf1dd5..b2a03faa7 100644 --- a/docs/user/reference/cli/azldev_completion_fish.md +++ b/docs/user/reference/cli/azldev_completion_fish.md @@ -45,7 +45,6 @@ azldev completion fish [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_powershell.md b/docs/user/reference/cli/azldev_completion_powershell.md index 6565209e0..24ffeb519 100644 --- a/docs/user/reference/cli/azldev_completion_powershell.md +++ b/docs/user/reference/cli/azldev_completion_powershell.md @@ -42,7 +42,6 @@ azldev completion powershell [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_zsh.md b/docs/user/reference/cli/azldev_completion_zsh.md index aa7d5d7a4..b1e9fa460 100644 --- a/docs/user/reference/cli/azldev_completion_zsh.md +++ b/docs/user/reference/cli/azldev_completion_zsh.md @@ -56,7 +56,6 @@ azldev completion zsh [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component.md b/docs/user/reference/cli/azldev_component.md index 621d10121..0ef76adda 100644 --- a/docs/user/reference/cli/azldev_component.md +++ b/docs/user/reference/cli/azldev_component.md @@ -35,7 +35,6 @@ components defined in the project configuration. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_add.md b/docs/user/reference/cli/azldev_component_add.md index 3b0b66fd0..53fb56931 100644 --- a/docs/user/reference/cli/azldev_component_add.md +++ b/docs/user/reference/cli/azldev_component_add.md @@ -48,7 +48,6 @@ azldev component add [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_build.md b/docs/user/reference/cli/azldev_component_build.md index 278ab3f6c..8392d9814 100644 --- a/docs/user/reference/cli/azldev_component_build.md +++ b/docs/user/reference/cli/azldev_component_build.md @@ -80,7 +80,6 @@ azldev component build [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_changed.md b/docs/user/reference/cli/azldev_component_changed.md index 581734e5d..c7a585d4a 100644 --- a/docs/user/reference/cli/azldev_component_changed.md +++ b/docs/user/reference/cli/azldev_component_changed.md @@ -14,8 +14,7 @@ change). For components listed in lockless-components.txt beside the current checkout's top-level azldev.toml, compare resolved project build inputs at both refs instead -of stored locks. The global --without-lockfile flag selects that comparison for -every component. +of stored locks. This is useful for CI/CD pipelines to determine which components need to be rebuilt or have their lookaside tarballs re-uploaded after a PR merge. @@ -75,7 +74,6 @@ azldev component changed [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_diff-sources.md b/docs/user/reference/cli/azldev_component_diff-sources.md index 4bcb11e90..4617a56f7 100644 --- a/docs/user/reference/cli/azldev_component_diff-sources.md +++ b/docs/user/reference/cli/azldev_component_diff-sources.md @@ -42,7 +42,6 @@ azldev component diff-sources [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_history.md b/docs/user/reference/cli/azldev_component_history.md index 8ac935060..6dea6264b 100644 --- a/docs/user/reference/cli/azldev_component_history.md +++ b/docs/user/reference/cli/azldev_component_history.md @@ -70,7 +70,6 @@ azldev component history [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_list.md b/docs/user/reference/cli/azldev_component_list.md index f0cdcfd7c..448d1e123 100644 --- a/docs/user/reference/cli/azldev_component_list.md +++ b/docs/user/reference/cli/azldev_component_list.md @@ -57,7 +57,6 @@ azldev component list [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_metadata.md b/docs/user/reference/cli/azldev_component_metadata.md index 35b0dc208..130b85186 100644 --- a/docs/user/reference/cli/azldev_component_metadata.md +++ b/docs/user/reference/cli/azldev_component_metadata.md @@ -83,7 +83,6 @@ azldev component metadata [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_prepare-sources.md b/docs/user/reference/cli/azldev_component_prepare-sources.md index 45f9f8cb7..e17685d77 100644 --- a/docs/user/reference/cli/azldev_component_prepare-sources.md +++ b/docs/user/reference/cli/azldev_component_prepare-sources.md @@ -63,7 +63,6 @@ azldev component prepare-sources [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_query.md b/docs/user/reference/cli/azldev_component_query.md index 17f212c53..12c3ca12c 100644 --- a/docs/user/reference/cli/azldev_component_query.md +++ b/docs/user/reference/cli/azldev_component_query.md @@ -62,7 +62,6 @@ azldev component query [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_refresh-upstream-commit.md b/docs/user/reference/cli/azldev_component_refresh-upstream-commit.md index aaebe323c..f8467899e 100644 --- a/docs/user/reference/cli/azldev_component_refresh-upstream-commit.md +++ b/docs/user/reference/cli/azldev_component_refresh-upstream-commit.md @@ -15,10 +15,9 @@ files before component-specific TOML configuration so subsequent commands use the generated commit unless the component configuration explicitly overrides it. -Without --without-lockfile, only components listed in lockless-components.txt +Only components listed in lockless-components.txt beside the top-level azldev.toml are refreshed. Other components are skipped -and their generated TOML files are preserved. With the global flag, every -component uses the lockless workflow. +and their generated TOML files are preserved. All TOML configuration is resolved before the source type is checked. Selected components whose effective spec.type is not "upstream" do not contact an @@ -92,7 +91,6 @@ azldev component refresh-upstream-commit [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index c1c3d1ca2..009afab51 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -24,7 +24,7 @@ Render mode is selected per component by lockless-components.txt beside the top-level azldev.toml. Each nonempty line names one component to render without its lock file, using configured upstream commits and their TOML history instead. All other components use lock files. A missing or empty file means all components -use lock files. The global --without-lockfile flag makes every component lockless. +use lock files. When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to @@ -89,7 +89,6 @@ azldev component render [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_test.md b/docs/user/reference/cli/azldev_component_test.md index e316666e3..363cc8870 100644 --- a/docs/user/reference/cli/azldev_component_test.md +++ b/docs/user/reference/cli/azldev_component_test.md @@ -109,7 +109,6 @@ azldev component test COMPONENT [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_update.md b/docs/user/reference/cli/azldev_component_update.md index bd5d89e05..acd88e290 100644 --- a/docs/user/reference/cli/azldev_component_update.md +++ b/docs/user/reference/cli/azldev_component_update.md @@ -88,7 +88,6 @@ azldev component update [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_config.md b/docs/user/reference/cli/azldev_config.md index 6eed54b63..e62fe0781 100644 --- a/docs/user/reference/cli/azldev_config.md +++ b/docs/user/reference/cli/azldev_config.md @@ -32,7 +32,6 @@ JSON schema used for validating TOML config files. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_config_dump.md b/docs/user/reference/cli/azldev_config_dump.md index 7136efc12..2f9860bc1 100644 --- a/docs/user/reference/cli/azldev_config_dump.md +++ b/docs/user/reference/cli/azldev_config_dump.md @@ -52,7 +52,6 @@ azldev config dump [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_config_generate-schema.md b/docs/user/reference/cli/azldev_config_generate-schema.md index 2f4eece5a..d92980891 100644 --- a/docs/user/reference/cli/azldev_config_generate-schema.md +++ b/docs/user/reference/cli/azldev_config_generate-schema.md @@ -46,7 +46,6 @@ azldev config generate-schema [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs.md b/docs/user/reference/cli/azldev_docs.md index 787033cbb..251289417 100644 --- a/docs/user/reference/cli/azldev_docs.md +++ b/docs/user/reference/cli/azldev_docs.md @@ -32,7 +32,6 @@ command tree, suitable for inclusion in the user guide. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_agent.md b/docs/user/reference/cli/azldev_docs_agent.md index fd1c785b0..e72765620 100644 --- a/docs/user/reference/cli/azldev_docs_agent.md +++ b/docs/user/reference/cli/azldev_docs_agent.md @@ -34,7 +34,6 @@ reference so that agents always load the guidance that ships with the binary. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_agent_install.md b/docs/user/reference/cli/azldev_docs_agent_install.md index f6fd83574..638938eef 100644 --- a/docs/user/reference/cli/azldev_docs_agent_install.md +++ b/docs/user/reference/cli/azldev_docs_agent_install.md @@ -32,8 +32,7 @@ Directory paths in the emitted content (such as the lock, generated upstream-commit, and rendered-spec directories) are resolved from the loaded azldev.toml, falling back to azldev's built-in defaults when no configuration is found. By default, the content includes both lock-file and lockless workflows -selected per component by lockless-components.txt. Pass --without-lockfile to -describe an entirely lockless project. The bindings reflect +selected per component by lockless-components.txt. The bindings reflect the project azldev runs in, so pair --output-dir with -C pointing at the target repository when scaffolding a different repo. @@ -75,7 +74,6 @@ azldev docs agent install [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_agent_show.md b/docs/user/reference/cli/azldev_docs_agent_show.md index 11d32383c..18a5dad11 100644 --- a/docs/user/reference/cli/azldev_docs_agent_show.md +++ b/docs/user/reference/cli/azldev_docs_agent_show.md @@ -51,7 +51,6 @@ azldev docs agent show [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_markdown.md b/docs/user/reference/cli/azldev_docs_markdown.md index a86b59b4d..4e70f63bc 100644 --- a/docs/user/reference/cli/azldev_docs_markdown.md +++ b/docs/user/reference/cli/azldev_docs_markdown.md @@ -53,7 +53,6 @@ azldev docs markdown [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image.md b/docs/user/reference/cli/azldev_image.md index ea678cf94..75579f3ca 100644 --- a/docs/user/reference/cli/azldev_image.md +++ b/docs/user/reference/cli/azldev_image.md @@ -33,7 +33,6 @@ can be customized using Azure Linux Image Customizer. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_boot.md b/docs/user/reference/cli/azldev_image_boot.md index 143536805..24ccc97bc 100644 --- a/docs/user/reference/cli/azldev_image_boot.md +++ b/docs/user/reference/cli/azldev_image_boot.md @@ -97,7 +97,6 @@ azldev image boot [IMAGE_NAME] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_build.md b/docs/user/reference/cli/azldev_image_build.md index 3c89f6b7a..d2aa0c8bf 100644 --- a/docs/user/reference/cli/azldev_image_build.md +++ b/docs/user/reference/cli/azldev_image_build.md @@ -58,7 +58,6 @@ azldev image build [image-name] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_customize.md b/docs/user/reference/cli/azldev_image_customize.md index 929c4f10b..a1125258d 100644 --- a/docs/user/reference/cli/azldev_image_customize.md +++ b/docs/user/reference/cli/azldev_image_customize.md @@ -56,7 +56,6 @@ azldev image customize [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_inject-files.md b/docs/user/reference/cli/azldev_image_inject-files.md index 748bde7f4..7dcd71ebb 100644 --- a/docs/user/reference/cli/azldev_image_inject-files.md +++ b/docs/user/reference/cli/azldev_image_inject-files.md @@ -47,7 +47,6 @@ azldev image inject-files [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_list.md b/docs/user/reference/cli/azldev_image_list.md index a1270bbd3..d22c8b4a4 100644 --- a/docs/user/reference/cli/azldev_image_list.md +++ b/docs/user/reference/cli/azldev_image_list.md @@ -49,7 +49,6 @@ azldev image list [image-name-pattern...] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_test.md b/docs/user/reference/cli/azldev_image_test.md index e2c1ca54e..cb5e9bdbe 100644 --- a/docs/user/reference/cli/azldev_image_test.md +++ b/docs/user/reference/cli/azldev_image_test.md @@ -82,7 +82,6 @@ azldev image test IMAGE_NAME [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_package.md b/docs/user/reference/cli/azldev_package.md index 678421bcc..fc4e5a176 100644 --- a/docs/user/reference/cli/azldev_package.md +++ b/docs/user/reference/cli/azldev_package.md @@ -33,7 +33,6 @@ publish channel assignments derived from package groups and component overrides. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_package_list.md b/docs/user/reference/cli/azldev_package_list.md index e6165273e..39ce75870 100644 --- a/docs/user/reference/cli/azldev_package_list.md +++ b/docs/user/reference/cli/azldev_package_list.md @@ -82,7 +82,6 @@ azldev package list [package-name...] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_project.md b/docs/user/reference/cli/azldev_project.md index fef53ace7..ea647dae6 100644 --- a/docs/user/reference/cli/azldev_project.md +++ b/docs/user/reference/cli/azldev_project.md @@ -32,7 +32,6 @@ as an Azure Linux project with a basic configuration. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_project_init.md b/docs/user/reference/cli/azldev_project_init.md index 13be79456..49de788c7 100644 --- a/docs/user/reference/cli/azldev_project_init.md +++ b/docs/user/reference/cli/azldev_project_init.md @@ -43,7 +43,6 @@ azldev project init [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_project_new.md b/docs/user/reference/cli/azldev_project_new.md index b494e46ee..e87182de9 100644 --- a/docs/user/reference/cli/azldev_project_new.md +++ b/docs/user/reference/cli/azldev_project_new.md @@ -46,7 +46,6 @@ azldev project new PATH [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_repo.md b/docs/user/reference/cli/azldev_repo.md index ccbc3f5ee..7d4091164 100644 --- a/docs/user/reference/cli/azldev_repo.md +++ b/docs/user/reference/cli/azldev_repo.md @@ -33,7 +33,6 @@ under one or more URL prefixes. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_repo_compare.md b/docs/user/reference/cli/azldev_repo_compare.md index 7053a3f19..dcb59b9b6 100644 --- a/docs/user/reference/cli/azldev_repo_compare.md +++ b/docs/user/reference/cli/azldev_repo_compare.md @@ -51,7 +51,6 @@ azldev repo compare [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_repo_query.md b/docs/user/reference/cli/azldev_repo_query.md index 9086aa1c9..f0c5cc540 100644 --- a/docs/user/reference/cli/azldev_repo_query.md +++ b/docs/user/reference/cli/azldev_repo_query.md @@ -74,7 +74,6 @@ azldev repo query [flags] -- -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_version.md b/docs/user/reference/cli/azldev_version.md index 5ebddbcdd..27febae21 100644 --- a/docs/user/reference/cli/azldev_version.md +++ b/docs/user/reference/cli/azldev_version.md @@ -37,7 +37,6 @@ azldev version -O json -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 477f16a86..f289122b5 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -151,8 +151,7 @@ The `[components..render]` section controls rendering behavior for a compo Lock mode for every command is selected by `lockless-components.txt` beside the top-level `azldev.toml`: listed components use the lockless workflow, and unlisted components use lock files. A missing or empty list selects lock-file -behavior for every component. The global `--without-lockfile` flag overrides -the list and makes every component lockless. See +behavior for every component. List every component for a fully lockless project. See [per-component lock modes](../../how-to/preview-without-lockfile.md#select-lock-mode-per-component). | Field | TOML Key | Type | Required | Description | diff --git a/docs/user/reference/config/config-file.md b/docs/user/reference/config/config-file.md index 1c24f7704..ffc719e09 100644 --- a/docs/user/reference/config/config-file.md +++ b/docs/user/reference/config/config-file.md @@ -39,7 +39,7 @@ Glob patterns that match no files are silently ignored. Literal filenames (no wi Includes are resolved recursively — included files can themselves declare further includes. For a detailed explanation of load order and merge semantics, see [Configuration System](../../explanation/config-system.md). -> **Preview:** For components listed in `lockless-components.txt` beside the top-level `azldev.toml`, or for every component with `--without-lockfile`, include generated upstream-commit files before component-specific TOML configuration so later definitions supply the remaining `spec` fields and may explicitly override a generated pin. See [Preview the Lock-File-Free Mode](../../how-to/preview-without-lockfile.md). +> **Preview:** For components listed in `lockless-components.txt` beside the top-level `azldev.toml`, include generated upstream-commit files before component-specific TOML configuration so later definitions supply the remaining `spec` fields and may explicitly override a generated pin. See [Preview the Lock-File-Free Mode](../../how-to/preview-without-lockfile.md). ## Minimal Example diff --git a/internal/app/azldev/agentskill/agentskill.go b/internal/app/azldev/agentskill/agentskill.go index 2f0ab5044..6d43365b4 100644 --- a/internal/app/azldev/agentskill/agentskill.go +++ b/internal/app/azldev/agentskill/agentskill.go @@ -211,8 +211,8 @@ type Catalog struct { withoutLockfile bool } -// NewCatalog returns the catalog for the selected mode. Pass true to describe -// lock-file-free mode, as selected by the global '--without-lockfile' flag. +// NewCatalog returns the default mixed-workflow catalog, or an internal +// all-lockless variant when withoutLockfile is true. The CLI uses the default. func NewCatalog(withoutLockfile bool) Catalog { return Catalog{withoutLockfile: withoutLockfile} } diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index c0c13e255..8bddc7993 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -36,8 +36,7 @@ images assemble RPMs into bootable artifacts. `lockless-components.txt` beside the top-level `azldev.toml` selects the lockless workflow per component for all commands. Listed components ignore locks and use -generated upstream-commit TOML instead. All others use locks. The global -`--without-lockfile` flag makes every component lockless. +generated upstream-commit TOML instead. All others use locks. Top-level commands in this build: diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index fb7c3366d..600b7886e 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -99,7 +99,7 @@ used. `%autorelease` behavior is unchanged on either path. azldev reads `lockless-components.txt` beside the top-level `azldev.toml`. Listed component names use the lockless workflow for every command; all others use lock files. A missing or empty list means all components use -locks. The global `--without-lockfile` flag makes every component lockless. +locks. List every component to use a fully lockless workflow. Use `comp refresh-upstream-commit` for listed components and `comp update` for unlisted components; each command skips the other mode. diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 8309c8c56..62e26c2c3 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -41,9 +41,8 @@ Add `-O json` for machine-readable output when debugging. ## Finalizing component changes Select this workflow by listing the component's exact name in -`lockless-components.txt` beside the top-level `azldev.toml`, or pass -`--without-lockfile` to make every component lockless. Without the flag, -refresh skips unlisted components and preserves their generated TOML files. +`lockless-components.txt` beside the top-level `azldev.toml`. +Refresh skips unlisted components and preserves their generated TOML files. For edits that do not change source resolution, render and commit the result. The generated upstream commit TOML does not change: diff --git a/internal/app/azldev/agentskill/content/update-component.md.tmpl b/internal/app/azldev/agentskill/content/update-component.md.tmpl index cce79005a..d3ca6690f 100644 --- a/internal/app/azldev/agentskill/content/update-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/update-component.md.tmpl @@ -14,7 +14,6 @@ those change, the lock is stale. This workflow applies only to components not listed in `lockless-components.txt` beside the top-level `azldev.toml`. Listed components use the `azldev-refresh-upstream-commit` skill instead; `comp update` skips them. -The global `--without-lockfile` flag makes every component lockless. ## When to run `update` diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 42467b659..aef27efce 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -89,7 +89,7 @@ calculation = "manual" azldev reads `lockless-components.txt` beside the top-level `azldev.toml`. Listed component names use the lockless workflow for every command; all others use lock files. A missing or empty list means all components use -locks. The global `--without-lockfile` flag makes every component lockless. +locks. List every component to use a fully lockless workflow. Use `comp refresh-upstream-commit` for lockless components. `render.skip-file-filter = true` keeps all source and patch files during render. diff --git a/internal/app/azldev/agentskill/doc.go b/internal/app/azldev/agentskill/doc.go index 49f193bcf..9d5e297a3 100644 --- a/internal/app/azldev/agentskill/doc.go +++ b/internal/app/azldev/agentskill/doc.go @@ -62,9 +62,8 @@ // // # Modes // -// A [Catalog] selects the content for one of azldev's two modes. The default mode -// documents the lock-file workflow; the mode selected by the global -// '--without-lockfile' flag documents the generated upstream-commit workflow. Skills, +// The CLI always uses the default [Catalog], which documents both per-component +// workflows. An internal all-lockless variant is also available. Skills, // instruction files, and emitted layout are shared; templates under // 'content/withoutlockfile' replace the same-named default template, and a small // registry replaces the skills whose subject differs. diff --git a/internal/app/azldev/app.go b/internal/app/azldev/app.go index f9cfda1ae..0932e5855 100644 --- a/internal/app/azldev/app.go +++ b/internal/app/azldev/app.go @@ -11,7 +11,6 @@ import ( "os" "os/signal" "path/filepath" - "strconv" "strings" "time" @@ -54,7 +53,6 @@ type App struct { commandPermissiveConfig bool configFiles []string colorMode ColorMode - withoutLockfile bool // Root command for the CLI. cmd cobra.Command @@ -191,8 +189,6 @@ func (app *App) registerGlobalFlags() { "output colorization mode {always, auto, never}") app.cmd.PersistentFlags().BoolVar(&app.permissiveConfigParsing, "permissive-config", false, "do not fail on unknown fields in TOML config files") - app.cmd.PersistentFlags().BoolVar(&app.withoutLockfile, "without-lockfile", false, - "preview: track resolved upstream commits in generated config instead of lock files") } // addAdvancedCommandHint embeds a hint about the hidden "advanced" command group @@ -426,7 +422,6 @@ func (a *App) initializeEnvOptions() *EnvOptions { envOptions.Interfaces.FileSystemFactory = a.fsFactory envOptions.Interfaces.OSEnvFactory = a.osEnvFactory envOptions.DryRunnable = NewAppDryRunnable(a.dryRun) - envOptions.WithoutLockfile = a.withoutLockfile return &envOptions } @@ -488,68 +483,16 @@ func setEventListener(stdioLogger *slog.Logger, quiet, verbose bool, envOptions return nil } -// PreParseGlobalFlags hand-parses the global flags that must be known before commands -// are registered and configuration is loaded. It is safe to call more than once with the +// PreParseGlobalFlags hand-parses the global flags needed to load configuration. +// It is safe to call more than once with the // same arguments; each call fully recomputes the pre-parsed state. -// -// Command registration is mode-sensitive, so the CLI entry point calls this before it -// registers commands; [App.Execute] calls it again so that an App executed directly -// (e.g. from a test) behaves identically. func (a *App) PreParseGlobalFlags(args []string) { // Reset accumulating state so repeated calls are idempotent. a.configFiles = nil - a.withoutLockfile = parseWithoutLockfileFlag(args) a.handParseConfigFlags(args) } -// WithoutLockfile reports whether the preview lock-file-free mode was requested via the -// global '--without-lockfile' flag. Valid only after [App.PreParseGlobalFlags] has run. -func (a *App) WithoutLockfile() bool { - return a.withoutLockfile -} - -// withoutLockfileFlagName is the global flag that selects lock-file-free mode. -const withoutLockfileFlagName = "--without-lockfile" - -// parseWithoutLockfileFlag hand-parses the global '--without-lockfile' boolean flag. -// -// Only exact '--without-lockfile' and '--without-lockfile=' tokens are -// recognized, and scanning stops at the '--' terminator so that positional text -// (for example a mock command line) is never mistaken for the flag. An invalid or -// missing value is treated the same way cobra treats it: '--without-lockfile' alone -// enables the mode, and an unparseable '=' leaves the mode disabled so the -// final cobra parse reports the error. -func parseWithoutLockfileFlag(args []string) bool { - withoutLockfile := false - - for _, arg := range args { - if arg == "--" { - break - } - - if arg == withoutLockfileFlagName { - withoutLockfile = true - - continue - } - - value, found := strings.CutPrefix(arg, withoutLockfileFlagName+"=") - if !found { - continue - } - - parsed, err := strconv.ParseBool(value) - if err != nil { - return false - } - - withoutLockfile = parsed - } - - return withoutLockfile -} - // Hand-parses a few critical configuration flags from the command line -- just enough to // find the project and load configuration so we can properly use cobra facilities to // parse the full command line. @@ -629,7 +572,7 @@ func (a *App) findAndLoadConfig(tempDirPath string, extraConfigFiles []string) ( tempDirPath, extraConfigFiles, a.permissiveConfigEnabled(), - a.withoutLockfile, + false, /*withoutLockfile*/ ) if err != nil { return projectDir, config, fmt.Errorf("failed to load project configuration:\n%w", err) diff --git a/internal/app/azldev/app_test.go b/internal/app/azldev/app_test.go index 64ac1397b..1b5af8648 100644 --- a/internal/app/azldev/app_test.go +++ b/internal/app/azldev/app_test.go @@ -238,66 +238,31 @@ func TestApp_ConcurrencyOption(t *testing.T) { } } -func TestApp_WithoutLockfileOption(t *testing.T) { +func TestApp_RejectsRemovedWithoutLockfileFlag(t *testing.T) { testCases := []struct { - name string - args []string - expected bool + name string + args []string }{ - {name: "absent", args: []string{"test-cmd"}, expected: false}, - {name: "bare", args: []string{"--without-lockfile", "test-cmd"}, expected: true}, - {name: "explicit true", args: []string{"--without-lockfile=true", "test-cmd"}, expected: true}, - {name: "explicit false", args: []string{"--without-lockfile=false", "test-cmd"}, expected: false}, - {name: "after command", args: []string{"test-cmd", "--without-lockfile"}, expected: true}, - {name: "positional lookalike", args: []string{"test-cmd", "--", "--without-lockfile"}, expected: false}, - {name: "prefix lookalike", args: []string{"test-cmd", "--without-lockfiles"}, expected: false}, + {name: "bare", args: []string{"--without-lockfile", "test-cmd"}}, + {name: "explicit true", args: []string{"--without-lockfile=true", "test-cmd"}}, + {name: "explicit false", args: []string{"--without-lockfile=false", "test-cmd"}}, + {name: "after command", args: []string{"test-cmd", "--without-lockfile"}}, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { app := createTestApp(t) - - app.PreParseGlobalFlags(testCase.args) - - assert.Equal(t, testCase.expected, app.WithoutLockfile()) - }) - } -} - -func TestApp_WithoutLockfileOption_ReachesEnv(t *testing.T) { - testCases := []struct { - name string - args []string - expected bool - }{ - {name: "default", args: []string{"test-cmd"}, expected: false}, - {name: "enabled", args: []string{"--without-lockfile", "test-cmd"}, expected: true}, - {name: "disabled", args: []string{"--without-lockfile=false", "test-cmd"}, expected: false}, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - app := createTestApp(t) - ran := false cmd := &cobra.Command{ Use: "test-cmd", - RunE: func(cmd *cobra.Command, _ []string) error { - env, err := azldev.GetEnvFromCommand(cmd) - require.NoError(t, err) - - assert.Equal(t, testCase.expected, env.WithoutLockfile()) - + Run: func(_ *cobra.Command, _ []string) { ran = true - - return nil }, } - app.AddTopLevelCommand(cmd) - assert.Zero(t, app.Execute(testCase.args)) - assert.True(t, ran) + assert.NotZero(t, app.Execute(testCase.args)) + assert.False(t, ran) }) } } @@ -306,7 +271,7 @@ func TestApp_WithoutLockfileOption_ReachesEnv(t *testing.T) { // registration) and by Execute; repeated calls must not accumulate state. func TestApp_PreParseGlobalFlags_Idempotent(t *testing.T) { app := createTestApp(t) - args := []string{"--without-lockfile", "--config-file", "extra.toml", "test-cmd"} + args := []string{"--config-file", "extra.toml", "test-cmd"} ran := false cmd := &cobra.Command{ @@ -315,7 +280,7 @@ func TestApp_PreParseGlobalFlags_Idempotent(t *testing.T) { env, err := azldev.GetEnvFromCommand(cmd) require.NoError(t, err) - assert.True(t, env.WithoutLockfile()) + assert.False(t, env.WithoutLockfile()) ran = true @@ -327,8 +292,6 @@ func TestApp_PreParseGlobalFlags_Idempotent(t *testing.T) { app.PreParseGlobalFlags(args) app.PreParseGlobalFlags(args) - assert.True(t, app.WithoutLockfile()) - // Execute pre-parses the same arguments a third time; a command that only // accumulated config files would fail to load its (nonexistent) extra file. assert.Zero(t, app.Execute(args)) diff --git a/internal/app/azldev/cmds/component/build.go b/internal/app/azldev/cmds/component/build.go index 3fff7e6c4..8c879ae9e 100644 --- a/internal/app/azldev/cmds/component/build.go +++ b/internal/app/azldev/cmds/component/build.go @@ -80,11 +80,11 @@ type ComponentBuildResults struct { RPMs []RPMResult `json:"rpms" table:"-"` } -func buildOnAppInit(app *azldev.App, parent *cobra.Command) { - parent.AddCommand(NewBuildCmd(cmdOptionsForApp(app)...)) +func buildOnAppInit(_ *azldev.App, parent *cobra.Command) { + parent.AddCommand(NewBuildCmd()) } -func NewBuildCmd(opts ...CmdOption) *cobra.Command { +func NewBuildCmd() *cobra.Command { // Fill out options defaults. options := &ComponentBuildOptions{ BuildEnvPolicy: BuildEnvPreserveOnFailure, @@ -132,7 +132,7 @@ builds can consume.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().BoolVarP(&options.ContinueOnError, "continue-on-error", "k", false, "Continue building when some components fail") cmd.Flags().BoolVar(&options.NoCheck, "no-check", false, "Skip package %check tests") diff --git a/internal/app/azldev/cmds/component/changed.go b/internal/app/azldev/cmds/component/changed.go index c70bbcb9a..3a90cb975 100644 --- a/internal/app/azldev/cmds/component/changed.go +++ b/internal/app/azldev/cmds/component/changed.go @@ -32,35 +32,12 @@ type ChangedComponentOptions struct { IncludeUnchanged bool } -func changedOnAppInit(app *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewChangedCmd(cmdOptionsForApp(app)...)) +func changedOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewChangedCmd()) } -// changedCommandLong returns the long help for 'component changed', which -// describes the comparison performed by the active mode. -func changedCommandLong(options cmdOptions) string { - if options.withoutLockfile { - return `Load the project configuration independently at two git refs and compare the -resolved component build inputs. Normal project TOML parsing is used at each -ref, including recursive includes, config merging, component defaults, and -generated upstream-commit TOMLs included by the project. - -For each selected component, the command reports whether the build inputs -changed: normalized component configuration, upstream commit or local -spec-directory contents, overlay source filenames and contents, and effective -distro release version. - -Documentation, publishing, test-selection, scheduling-hint, snapshot-time, and -checkout-path-only fields do not trigger a component change. For components -whose build inputs changed, the sourcesChange field separately compares the raw -committed sources manifests at the rendered spec directories configured by each -ref. All-component scans use the union of components in both refs, so added and -deleted components are reported without consulting the current checkout. - -This is useful for CI/CD pipelines to determine which components need to be -rebuilt or have their lookaside tarballs re-uploaded after a PR merge.` - } - +// changedCommandLong describes the per-component comparisons. +func changedCommandLong() string { return `Compare component lock files and rendered sources between two git refs to determine which components changed. For each component, reports whether its input fingerprint changed (any change). For components whose fingerprint @@ -69,8 +46,7 @@ change). For components listed in lockless-components.txt beside the current checkout's top-level azldev.toml, compare resolved project build inputs at both refs instead -of stored locks. The global --without-lockfile flag selects that comparison for -every component. +of stored locks. This is useful for CI/CD pipelines to determine which components need to be rebuilt or have their lookaside tarballs re-uploaded after a PR merge. @@ -83,14 +59,13 @@ detected via lock file presence in the compared refs when using -a.` } // NewChangedCmd constructs a [cobra.Command] for the "component changed" CLI subcommand. -func NewChangedCmd(opts ...CmdOption) *cobra.Command { +func NewChangedCmd() *cobra.Command { options := &ChangedComponentOptions{} - cmdOptions := newCmdOptions(opts...) cmd := &cobra.Command{ Use: "changed", Short: "Detect which components changed between two git refs", - Long: changedCommandLong(cmdOptions), + Long: changedCommandLong(), Example: ` # Show changed components between a branch and HEAD azldev component changed --from main -a @@ -110,7 +85,7 @@ func NewChangedCmd(opts ...CmdOption) *cobra.Command { ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().StringVar(&options.From, "from", "", "Git ref to compare from (required)") cmd.Flags().StringVar(&options.To, "to", "HEAD", "Git ref to compare to") @@ -121,9 +96,7 @@ func NewChangedCmd(opts ...CmdOption) *cobra.Command { // Hide inherited flag -- this command always skips lock validation since // it inspects historical locks at arbitrary refs. - if !cmdOptions.withoutLockfile { - _ = cmd.Flags().MarkHidden("skip-lock-validation") - } + _ = cmd.Flags().MarkHidden("skip-lock-validation") azldev.ExportAsReadOnlyMCPTool(cmd) diff --git a/internal/app/azldev/cmds/component/changed_test.go b/internal/app/azldev/cmds/component/changed_test.go index 0e0952ea6..bf33c9612 100644 --- a/internal/app/azldev/cmds/component/changed_test.go +++ b/internal/app/azldev/cmds/component/changed_test.go @@ -60,11 +60,9 @@ func TestChangedCmd_NoComponents(t *testing.T) { assert.Contains(t, err.Error(), "component not found") } -func TestNewChangedCmd_LongDescriptionByMode(t *testing.T) { - defaultCmd := componentcmds.NewChangedCmd() - assert.Contains(t, defaultCmd.Long, "Compare component lock files") - - withoutLockfileCmd := componentcmds.NewChangedCmd(componentcmds.WithoutLockfileFlags()) - assert.Contains(t, withoutLockfileCmd.Long, "Load the project configuration independently") - assert.NotContains(t, withoutLockfileCmd.Long, "lock file") +func TestNewChangedCmd_DocumentsPerComponentSelection(t *testing.T) { + cmd := componentcmds.NewChangedCmd() + assert.Contains(t, cmd.Long, "Compare component lock files") + assert.Contains(t, cmd.Long, "lockless-components.txt") + assert.Contains(t, cmd.Long, "compare resolved project build inputs") } diff --git a/internal/app/azldev/cmds/component/cmdoptions.go b/internal/app/azldev/cmds/component/cmdoptions.go index b347f3ca6..be0ecd424 100644 --- a/internal/app/azldev/cmds/component/cmdoptions.go +++ b/internal/app/azldev/cmds/component/cmdoptions.go @@ -4,59 +4,15 @@ package component import ( - "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/spf13/cobra" ) -// cmdOptions holds the mode-sensitive choices made when a component command is -// constructed. The zero value describes azldev's default (lock file) mode. -type cmdOptions struct { - // withoutLockfile omits lock-file-specific flags, matching the command surface - // exposed by the global '--without-lockfile' flag. - withoutLockfile bool -} - -// CmdOption customizes how a component command is constructed. -type CmdOption func(*cmdOptions) - -// WithoutLockfileFlags omits the lock-file-specific flags from a component command. -// Commands are registered with this option when the global '--without-lockfile' -// flag selects lock-file-free mode, so the flags that only lock files can honor are -// never offered. -func WithoutLockfileFlags() CmdOption { - return func(options *cmdOptions) { - options.withoutLockfile = true - } -} - -// newCmdOptions resolves the supplied command options. -func newCmdOptions(opts ...CmdOption) cmdOptions { - options := cmdOptions{} - for _, opt := range opts { - opt(&options) - } - - return options -} - // addComponentFilterOptions registers the component selection flags, including the -// lock-file flags that only apply in azldev's default mode. +// lock-validation flag honored by components using lock files. func addComponentFilterOptions( - cmd *cobra.Command, filter *components.ComponentFilter, options cmdOptions, + cmd *cobra.Command, filter *components.ComponentFilter, ) { components.AddComponentFilterOptionsToCommand(cmd, filter) - - if !options.withoutLockfile { - components.AddLockValidationFlagToCommand(cmd, filter) - } -} - -// cmdOptionsForApp returns the command options matching the app's selected mode. -func cmdOptionsForApp(app *azldev.App) []CmdOption { - if app.WithoutLockfile() { - return []CmdOption{WithoutLockfileFlags()} - } - - return nil + components.AddLockValidationFlagToCommand(cmd, filter) } diff --git a/internal/app/azldev/cmds/component/component.go b/internal/app/azldev/cmds/component/component.go index 6cca90609..dced09151 100644 --- a/internal/app/azldev/cmds/component/component.go +++ b/internal/app/azldev/cmds/component/component.go @@ -37,17 +37,10 @@ components defined in the project configuration.`, testOnAppInit(app, cmd) metadataOnAppInit(app, cmd) - // Mixed projects need both state writers. The global override keeps the - // existing entirely lockless command surface and legacy no-ops. - if app.WithoutLockfile() { - legacyOnAppInit(app, cmd) - refreshUpstreamCommitOnAppInit(app, cmd) - } else { - historyOnAppInit(app, cmd) - queryOnAppInit(app, cmd) - updateOnAppInit(app, cmd) - refreshUpstreamCommitOnAppInit(app, cmd) - } + historyOnAppInit(app, cmd) + queryOnAppInit(app, cmd) + updateOnAppInit(app, cmd) + refreshUpstreamCommitOnAppInit(app, cmd) return cmd } diff --git a/internal/app/azldev/cmds/component/component_test.go b/internal/app/azldev/cmds/component/component_test.go index c8ae56ebc..907d702ee 100644 --- a/internal/app/azldev/cmds/component/component_test.go +++ b/internal/app/azldev/cmds/component/component_test.go @@ -4,7 +4,6 @@ package component_test import ( - "slices" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" @@ -31,54 +30,26 @@ func TestOnAppInit(t *testing.T) { assert.Contains(t, topLevelCommandNames, "component") } -func TestOnAppInit_CommandsByMode(t *testing.T) { - testCases := []struct { - name string - args []string - refreshExpected bool - }{ - {name: "default mode", args: nil, refreshExpected: true}, - {name: "lock-file-free mode", args: []string{"--without-lockfile"}, refreshExpected: true}, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - app := azldev.NewApp(opctx_test.NewMockFileSystemFactory(ctrl), opctx_test.NewMockOSEnvFactory(ctrl)) - - app.PreParseGlobalFlags(testCase.args) - component.OnAppInit(app) - - commandNames, err := app.CommandNames("component") - require.NoError(t, err) +func TestOnAppInit_RegistersBothWorkflows(t *testing.T) { + ctrl := gomock.NewController(t) + app := azldev.NewApp(opctx_test.NewMockFileSystemFactory(ctrl), opctx_test.NewMockOSEnvFactory(ctrl)) + component.OnAppInit(app) - assert.Equal(t, testCase.refreshExpected, - slices.Contains(commandNames, "refresh-upstream-commit")) + commandNames, err := app.CommandNames("component") + require.NoError(t, err) - // The lock-file command names stay reachable in lock-file-free mode, - // where they are registered as hidden no-ops for compatibility. - for _, name := range []string{"history", "query", "update"} { - assert.Contains(t, commandNames, name) - } - }) + for _, name := range []string{"history", "query", "update", "refresh-upstream-commit"} { + assert.Contains(t, commandNames, name) } } -func TestNewComponentCommands_LockValidationFlagByMode(t *testing.T) { - // The lock-file consistency flags only mean something when lock files are in - // play, so lock-file-free mode leaves them unregistered. +func TestNewComponentCommands_LockValidationFlags(t *testing.T) { assert.NotNil(t, component.NewBuildCmd().Flags().Lookup("skip-lock-validation")) assert.NotNil(t, component.NewRenderCmd().Flags().Lookup("skip-lock-validation")) assert.NotNil(t, component.NewComponentListCommand().Flags().Lookup("skip-lock-validation")) assert.NotNil(t, component.NewHistoryCmd().Flags().Lookup("skip-lock-validation")) assert.NotNil(t, component.NewComponentQueryCommand().Flags().Lookup("skip-lock-validation")) assert.NotNil(t, component.NewUpdateCmd().Flags().Lookup("skip-lock-validation")) - - withoutLockfile := component.WithoutLockfileFlags() - assert.Nil(t, component.NewBuildCmd(withoutLockfile).Flags().Lookup("skip-lock-validation")) - assert.NotNil(t, component.NewRenderCmd(withoutLockfile).Flags().Lookup("skip-lock-validation"), - "render selects lock behavior per component, independently of the global mode") - assert.Nil(t, component.NewComponentListCommand(withoutLockfile).Flags().Lookup("skip-lock-validation")) } func TestLocklessComponents_SkipLegacyHistoryAndQuery(t *testing.T) { diff --git a/internal/app/azldev/cmds/component/diffsources.go b/internal/app/azldev/cmds/component/diffsources.go index 7fd728050..bd76370d1 100644 --- a/internal/app/azldev/cmds/component/diffsources.go +++ b/internal/app/azldev/cmds/component/diffsources.go @@ -29,12 +29,12 @@ type DiffSourcesOptions struct { OutputFile string } -func diffSourcesOnAppInit(app *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewDiffSourcesCmd(cmdOptionsForApp(app)...)) +func diffSourcesOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewDiffSourcesCmd()) } // NewDiffSourcesCmd constructs a [cobra.Command] for the "component diff-sources" CLI subcommand. -func NewDiffSourcesCmd(opts ...CmdOption) *cobra.Command { +func NewDiffSourcesCmd() *cobra.Command { var options DiffSourcesOptions var cmd *cobra.Command @@ -54,7 +54,7 @@ overlays to the copy and displays the resulting diff between the two trees.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().StringVar(&options.OutputFile, "output-file", "", "write the diff output to a file instead of stdout") diff --git a/internal/app/azldev/cmds/component/history.go b/internal/app/azldev/cmds/component/history.go index 274554aa4..d8b87bf6c 100644 --- a/internal/app/azldev/cmds/component/history.go +++ b/internal/app/azldev/cmds/component/history.go @@ -110,7 +110,7 @@ hand-picking entries to document.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions{}) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().StringVar(&options.SharedTomlMode, "shared", sharedTomlModeShow, "How to report rows for components that share a TOML file with others: "+ diff --git a/internal/app/azldev/cmds/component/legacy.go b/internal/app/azldev/cmds/component/legacy.go deleted file mode 100644 index 656d9ba71..000000000 --- a/internal/app/azldev/cmds/component/legacy.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -package component - -import ( - "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" - "github.com/spf13/cobra" -) - -const ( - legacyUpdateMessage = "azldev component update no longer does anything and should no longer be used." - legacyHistoryMessage = "azldev component history no longer does anything and should no longer be used." - legacyQueryMessage = "azldev component query no longer does anything and should no longer be used." -) - -func legacyOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand( - newLegacyNoOpCmd("update", nil, legacyUpdateMessage), - newLegacyNoOpCmd("history", []string{"hist"}, legacyHistoryMessage), - newLegacyNoOpCmd("query", nil, legacyQueryMessage), - ) -} - -func newLegacyNoOpCmd(name string, aliases []string, message string) *cobra.Command { - cmd := &cobra.Command{ - Use: name, - Aliases: aliases, - Short: message, - Hidden: true, - DisableFlagParsing: true, - Args: cobra.ArbitraryArgs, - Run: func(cmd *cobra.Command, _ []string) { - cmd.Println(message) - }, - } - - azldev.ExcludeFromMarkdownDocs(cmd) - - return cmd -} diff --git a/internal/app/azldev/cmds/component/legacy_internal_test.go b/internal/app/azldev/cmds/component/legacy_internal_test.go deleted file mode 100644 index 50bac09e5..000000000 --- a/internal/app/azldev/cmds/component/legacy_internal_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -package component - -import ( - "bytes" - "testing" - - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestLegacyCommandsAreHiddenNoOps(t *testing.T) { - tests := []struct { - name string - command string - args []string - expected string - }{ - { - name: "update", - command: "update", - args: []string{"-a", "--bump", "curl"}, - expected: legacyUpdateMessage, - }, - { - name: "history", - command: "history", - args: []string{"-a", "--include-bare", "-O", "json"}, - expected: legacyHistoryMessage, - }, - { - name: "history alias", - command: "hist", - args: []string{"curl"}, - expected: legacyHistoryMessage, - }, - { - name: "query", - command: "query", - args: []string{"-p", "curl", "--arch", "aarch64", "-O", "json"}, - expected: legacyQueryMessage, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - parent := &cobra.Command{Use: "component"} - legacyOnAppInit(nil, parent) - - var output bytes.Buffer - parent.SetOut(&output) - parent.SetErr(&output) - parent.SetArgs(append([]string{test.command}, test.args...)) - - require.NoError(t, parent.Execute()) - assert.Equal(t, test.expected+"\n", output.String()) - - command, _, err := parent.Find([]string{test.command}) - require.NoError(t, err) - assert.True(t, command.Hidden) - assert.True(t, command.DisableFlagParsing) - }) - } -} diff --git a/internal/app/azldev/cmds/component/list.go b/internal/app/azldev/cmds/component/list.go index 4f0ba3ee1..271119346 100644 --- a/internal/app/azldev/cmds/component/list.go +++ b/internal/app/azldev/cmds/component/list.go @@ -19,12 +19,12 @@ type ListComponentOptions struct { ComponentFilter components.ComponentFilter } -func listOnAppInit(app *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewComponentListCommand(cmdOptionsForApp(app)...)) +func listOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewComponentListCommand()) } // Constructs a [cobra.Command] for "component list" CLI subcommand. -func NewComponentListCommand(opts ...CmdOption) *cobra.Command { +func NewComponentListCommand() *cobra.Command { options := &ListComponentOptions{} cmd := &cobra.Command{ @@ -55,14 +55,11 @@ Component name patterns support glob syntax (*, ?, []).`, azldev.ExportAsReadOnlyMCPTool(cmd) - cmdOptions := newCmdOptions(opts...) - addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions) + addComponentFilterOptions(cmd, &options.ComponentFilter) // List always skips lock validation (read-only), so the flag is // meaningless here. Hide it to avoid confusion. - if !cmdOptions.withoutLockfile { - _ = cmd.Flags().MarkHidden("skip-lock-validation") - } + _ = cmd.Flags().MarkHidden("skip-lock-validation") return cmd } diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index d55781037..f234712b5 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -29,11 +29,11 @@ type PrepareSourcesOptions struct { SkipSources bool } -func prepareOnAppInit(app *azldev.App, sourceCmd *cobra.Command) { - sourceCmd.AddCommand(NewPrepareSourcesCmd(cmdOptionsForApp(app)...)) +func prepareOnAppInit(_ *azldev.App, sourceCmd *cobra.Command) { + sourceCmd.AddCommand(NewPrepareSourcesCmd()) } -func NewPrepareSourcesCmd(opts ...CmdOption) *cobra.Command { +func NewPrepareSourcesCmd() *cobra.Command { var options PrepareSourcesOptions var cmd *cobra.Command @@ -67,7 +67,7 @@ Only one component may be selected at a time.`, }, } - addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().StringVarP(&options.OutputDir, "output-dir", "o", "", "output directory") _ = cmd.MarkFlagRequired("output-dir") diff --git a/internal/app/azldev/cmds/component/query.go b/internal/app/azldev/cmds/component/query.go index 000529df9..b6d097059 100644 --- a/internal/app/azldev/cmds/component/query.go +++ b/internal/app/azldev/cmds/component/query.go @@ -71,7 +71,7 @@ The rendered-specs-dir must exist on disk; if it doesn't, run ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions{}) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().Var(&options.Arch, "arch", "Target architecture passed to rpmspec via --target (x86_64, aarch64). "+ diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit.go b/internal/app/azldev/cmds/component/refreshupstreamcommit.go index a1be4f70a..be5119847 100644 --- a/internal/app/azldev/cmds/component/refreshupstreamcommit.go +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit.go @@ -54,10 +54,9 @@ files before component-specific TOML configuration so subsequent commands use the generated commit unless the component configuration explicitly overrides it. -Without --without-lockfile, only components listed in lockless-components.txt +Only components listed in lockless-components.txt beside the top-level azldev.toml are refreshed. Other components are skipped -and their generated TOML files are preserved. With the global flag, every -component uses the lockless workflow. +and their generated TOML files are preserved. All TOML configuration is resolved before the source type is checked. Selected components whose effective spec.type is not "upstream" do not contact an @@ -366,7 +365,7 @@ func refreshCheckOnlyResult( } return display, fmt.Errorf("upstream commit TOML files are stale; %s. "+ - "Run 'azldev --without-lockfile component refresh-upstream-commit -a' to refresh", + "Run 'azldev component refresh-upstream-commit -a' to refresh", strings.Join(parts, "; ")) } diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go index 9f42595a0..1bba0d451 100644 --- a/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go @@ -55,22 +55,12 @@ func TestRefreshUpstreamCommitCmd_CleansPinsThatInvalidateStrictConfig(t *testin testCases := []struct { name string componentConfig string - withoutLockfile bool }{ { - name: "removed component", - withoutLockfile: true, + name: "removed component", }, { - name: "converted to local", - withoutLockfile: true, - componentConfig: `[components.test-component.spec] -type = "local" -path = "../../specs/test-component.spec" -`, - }, - { - name: "listed component converted to local", + name: "converted to local", componentConfig: `[components.test-component.spec] type = "local" path = "../../specs/test-component.spec" @@ -80,7 +70,7 @@ path = "../../specs/test-component.spec" for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - testEnv := testutils.NewTestEnvWithoutLockfile(t) + testEnv := testutils.NewTestEnv(t) rootConfig := `includes = [ "base/upstream-commits/*.toml", @@ -103,10 +93,8 @@ release-ver = "1.0" fileperms.PublicFile, )) - if !testCase.withoutLockfile { - require.NoError(t, fileutils.WriteFile(testEnv.TestFS, "/project/lockless-components.txt", - []byte("test-component\n"), fileperms.PublicFile)) - } + require.NoError(t, fileutils.WriteFile(testEnv.TestFS, "/project/lockless-components.txt", + []byte("test-component\n"), fileperms.PublicFile)) if testCase.componentConfig != "" { require.NoError(t, fileutils.WriteFile( @@ -126,9 +114,6 @@ release-ver = "1.0" ) args := []string{"component", "refresh-upstream-commit", "--all-components"} - if testCase.withoutLockfile { - args = append([]string{"--without-lockfile"}, args...) - } app.PreParseGlobalFlags(args) componentcmds.OnAppInit(app) @@ -438,7 +423,7 @@ func TestRefreshUpstreamCommits_CheckOnly_StaleReturnsError(t *testing.T) { require.Error(t, err, "stale TOML must produce a non-nil error in --check-only mode") assert.Contains(t, err.Error(), "stale", "error message should mention staleness") assert.Contains(t, err.Error(), "curl", "error message should name the stale component") - assert.Contains(t, err.Error(), "azldev --without-lockfile component refresh-upstream-commit -a", + assert.Contains(t, err.Error(), "azldev component refresh-upstream-commit -a", "-a-scoped run should suggest the same -a invocation to refresh") // Results slice must be returned alongside the error so structured diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index c9ad8a758..349ee2987 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -46,7 +46,7 @@ func renderOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { } // NewRenderCmd constructs a [cobra.Command] for the "component render" CLI subcommand. -func NewRenderCmd(_ ...CmdOption) *cobra.Command { +func NewRenderCmd() *cobra.Command { var options RenderOptions var cmd *cobra.Command @@ -72,7 +72,7 @@ Render mode is selected per component by lockless-components.txt beside the top-level azldev.toml. Each nonempty line names one component to render without its lock file, using configured upstream commits and their TOML history instead. All other components use lock files. A missing or empty file means all components -use lock files. The global --without-lockfile flag makes every component lockless. +use lock files. When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to @@ -107,7 +107,7 @@ valid with -a.`, }, } - addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions{}) + addComponentFilterOptions(cmd, &options.ComponentFilter) addRenderOptions(cmd, &options) return cmd diff --git a/internal/app/azldev/cmds/component/render_modes_internal_test.go b/internal/app/azldev/cmds/component/render_modes_internal_test.go index a924d75f9..4bf14dfff 100644 --- a/internal/app/azldev/cmds/component/render_modes_internal_test.go +++ b/internal/app/azldev/cmds/component/render_modes_internal_test.go @@ -87,8 +87,8 @@ func TestRenderComponentsSelectsLockModeFromFile(t *testing.T) { name string withoutLockfile bool }{ - {name: "default global mode"}, - {name: "lockless global mode", withoutLockfile: true}, + {name: "project component list"}, + {name: "internal whole-project override", withoutLockfile: true}, } { t.Run(testCase.name, func(t *testing.T) { env := testutils.NewTestEnv(t) @@ -153,7 +153,7 @@ func TestRenderComponentsSelectsLockModeFromFile(t *testing.T) { require.Len(t, observed, 2) if testCase.withoutLockfile { - assert.Nil(t, observed["curl"], "the global flag overrides the component list") + assert.Nil(t, observed["curl"], "the internal override applies to every component") } else { require.NotNil(t, observed["curl"]) assert.Equal(t, "configured-commit", observed["curl"].UpstreamCommit) diff --git a/internal/app/azldev/cmds/component/update.go b/internal/app/azldev/cmds/component/update.go index 34a7b2a03..1652f1783 100644 --- a/internal/app/azldev/cmds/component/update.go +++ b/internal/app/azldev/cmds/component/update.go @@ -104,7 +104,7 @@ Cannot be combined with --bump.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions{}) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().BoolVar(&options.Bump, "bump", false, "increment the manual-rebuild counter to trigger a new release") diff --git a/internal/app/azldev/cmds/docs/agent.go b/internal/app/azldev/cmds/docs/agent.go index 793e1da49..bb65a46f7 100644 --- a/internal/app/azldev/cmds/docs/agent.go +++ b/internal/app/azldev/cmds/docs/agent.go @@ -35,8 +35,8 @@ type InstalledAgentFile struct { } // Called once when the app is initialized; registers the 'agent' command tree under 'docs'. -func agentOnAppInit(app *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(newAgentCmd(agentskill.NewCatalog(app.WithoutLockfile()))) +func agentOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(newAgentCmd(agentskill.NewCatalog(false))) } func newAgentCmd(catalog agentskill.Catalog) *cobra.Command { @@ -90,8 +90,7 @@ Directory paths in the emitted content (such as the lock, generated upstream-commit, and rendered-spec directories) are resolved from the loaded azldev.toml, falling back to azldev's built-in defaults when no configuration is found. By default, the content includes both lock-file and lockless workflows -selected per component by lockless-components.txt. Pass --without-lockfile to -describe an entirely lockless project. The bindings reflect +selected per component by lockless-components.txt. The bindings reflect the project azldev runs in, so pair --output-dir with -C pointing at the target repository when scaffolding a different repo.`, Example: ` # Write agent files into the current repository @@ -152,7 +151,7 @@ skill to list the available skills.`, return fmt.Errorf("failed to get command environment:\n%w", err) } - name, list, err := resolveShowSkill(agentSkillCatalog(env), skillName) + name, list, err := resolveShowSkill(agentskill.NewCatalog(false), skillName) if err != nil { return err } @@ -166,7 +165,7 @@ skill to list the available skills.`, return nil } - doc, err := agentSkillCatalog(env).SkillDocument(name, agentSkillParams(env, cmd.Root())) + doc, err := agentskill.NewCatalog(false).SkillDocument(name, agentSkillParams(env, cmd.Root())) if err != nil { return fmt.Errorf("failed to render azldev skill:\n%w", err) } @@ -194,7 +193,7 @@ func InstallAgentFiles( return nil, err } - files, err := agentSkillCatalog(env).Files(layout, agentSkillParams(env, rootCmd), options.Full) + files, err := agentskill.NewCatalog(false).Files(layout, agentSkillParams(env, rootCmd), options.Full) if err != nil { return nil, fmt.Errorf("failed to render azldev agent files:\n%w", err) } @@ -321,12 +320,6 @@ func emitMCPConfig( return InstalledAgentFile{Path: destPath, Written: true}, nil } -// agentSkillCatalog returns the skill catalog describing the mode azldev is running -// in, so the emitted and served content matches the commands the user actually has. -func agentSkillCatalog(env *azldev.Env) agentskill.Catalog { - return agentskill.NewCatalog(env != nil && env.WithoutLockfile()) -} - // agentSkillParams gathers the dynamic values injected into the emitted and served agent content, // including the target-repo bindings resolved from the loaded project configuration. func agentSkillParams(env *azldev.Env, rootCmd *cobra.Command) agentskill.Params { diff --git a/internal/app/azldev/core/testutils/testenv.go b/internal/app/azldev/core/testutils/testenv.go index cf913c8ff..678da9fdf 100644 --- a/internal/app/azldev/core/testutils/testenv.go +++ b/internal/app/azldev/core/testutils/testenv.go @@ -71,7 +71,7 @@ func NewTestEnv(t *testing.T) *TestEnv { // NewTestEnvWithoutLockfile creates a [TestEnv] configured for azldev's // lock-file-free mode: no lock directory is configured and the environment // reports [azldev.Env.WithoutLockfile], so lock population and lock validation -// are skipped exactly as they are under the global '--without-lockfile' flag. +// are skipped for every component via an internal environment override. func NewTestEnvWithoutLockfile(t *testing.T) *TestEnv { t.Helper() diff --git a/internal/app/azldev/env.go b/internal/app/azldev/env.go index f567590bd..35acd9f53 100644 --- a/internal/app/azldev/env.go +++ b/internal/app/azldev/env.go @@ -35,8 +35,8 @@ type EnvOptions struct { // WithoutLockfile selects the preview lock-file-free mode, in which // component state is tracked by generated upstream-commit config instead - // of per-component lock files. Set from the global '--without-lockfile' - // flag; false selects the default lock-file behavior. + // of per-component lock files. Used for internal historical comparisons; + // CLI operations select their modes from the project's component list. WithoutLockfile bool // Injected dependencies. @@ -283,8 +283,8 @@ func (env *Env) WithoutLockfile() bool { return env.withoutLockfile } -// ComponentWithoutLockfile reports the selected component's mode. The global -// '--without-lockfile' flag overrides the project's per-component list. +// ComponentWithoutLockfile reports the selected component's mode, honoring the +// internal whole-project override used by historical comparisons. func (env *Env) ComponentWithoutLockfile(name string) bool { return env.withoutLockfile || (env.config != nil && env.config.LocklessComponents[name]) } diff --git a/pkg/app/azldev_cli/azldev.go b/pkg/app/azldev_cli/azldev.go index 786e5df6b..1aa0b5633 100644 --- a/pkg/app/azldev_cli/azldev.go +++ b/pkg/app/azldev_cli/azldev.go @@ -39,20 +39,18 @@ func Main() { } // InstantiateApp constructs a new instance of the azldev CLI application with -// all subcommands registered for azldev's default (lock file) mode. +// all subcommands registered. func InstantiateApp() *azldev.App { return InstantiateAppForArgs(nil) } // InstantiateAppForArgs constructs a new instance of the azldev CLI application -// with the subcommands registered for the mode selected by args. Global flags -// that select a mode (such as '--without-lockfile') are hand-parsed from args -// before registration, because the registered command set differs by mode. +// with configuration flags pre-parsed from args and all subcommands registered. func InstantiateAppForArgs(args []string) *azldev.App { // Instantiate the main CLI application. app := azldev.NewApp(azldev.DefaultFileSystemFactory(), azldev.DefaultOSEnvFactory()) - // Resolve the global flags that command registration depends on. + // Pre-parse flags needed for configuration loading. app.PreParseGlobalFlags(args) // Give top level command packages an opportunity to register their commands (or in some cases, diff --git a/scenario/without_lockfile_test.go b/scenario/without_lockfile_test.go index 53df92409..b42a85838 100644 --- a/scenario/without_lockfile_test.go +++ b/scenario/without_lockfile_test.go @@ -22,12 +22,7 @@ import ( "github.com/stretchr/testify/require" ) -// These tests cover the global lockless override and per-component selection -// through 'lockless-components.txt'. - -// TestWithoutLockfile_GlobalFlagIsDocumented verifies that the preview flag is part -// of the CLI surface and defaults to off. -func TestWithoutLockfile_GlobalFlagIsDocumented(t *testing.T) { +func TestLocklessComponents_NoGlobalFlag(t *testing.T) { t.Parallel() if testing.Short() { @@ -37,66 +32,25 @@ func TestWithoutLockfile_GlobalFlagIsDocumented(t *testing.T) { results, err := cmdtest.NewScenarioTest("--help").Locally().Run(t) require.NoError(t, err) require.Zero(t, results.ExitCode) - assert.Contains(t, results.Stdout, "--without-lockfile") + assert.NotContains(t, results.Stdout, "--without-lockfile") } -// TestWithoutLockfile_ComponentCommandsByMode verifies that the component command -// set exposes both workflows by default and the lockless workflow with the flag. -func TestWithoutLockfile_ComponentCommandsByMode(t *testing.T) { +func TestLocklessComponents_BothWorkflowsAvailable(t *testing.T) { t.Parallel() if testing.Short() { t.Skip("skipping long test") } - testCases := []struct { - name string - args []string - expected []string - notExpected []string - }{ - { - name: "default mode", - args: []string{"component", "--help"}, - expected: []string{"update", "history", "query", "refresh-upstream-commit"}, - }, - { - name: "explicitly disabled", - args: []string{"--without-lockfile=false", "component", "--help"}, - expected: []string{"update", "history", "query", "refresh-upstream-commit"}, - }, - { - name: "lock-file-free mode", - args: []string{"--without-lockfile", "component", "--help"}, - expected: []string{"refresh-upstream-commit"}, - // The lock-file commands remain registered as hidden no-ops, so they - // must not be advertised in help. - notExpected: []string{"Refresh component lock files"}, - }, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - t.Parallel() - - results, err := cmdtest.NewScenarioTest(testCase.args...).Locally().Run(t) - require.NoError(t, err) - require.Zero(t, results.ExitCode) - - for _, expected := range testCase.expected { - assert.Contains(t, results.Stdout, expected) - } - - for _, notExpected := range testCase.notExpected { - assert.NotContains(t, results.Stdout, notExpected) - } - }) + results, err := cmdtest.NewScenarioTest("component", "--help").Locally().Run(t) + require.NoError(t, err) + require.Zero(t, results.ExitCode) + for _, name := range []string{"update", "history", "query", "refresh-upstream-commit"} { + assert.Contains(t, results.Stdout, name) } } -// TestWithoutLockfile_BuildFlagsByMode verifies that the lock-file-only component -// flags are registered only in the default mode. -func TestWithoutLockfile_BuildFlagsByMode(t *testing.T) { +func TestLocklessComponents_LockValidationFlagAvailable(t *testing.T) { t.Parallel() if testing.Short() { @@ -108,34 +62,9 @@ func TestWithoutLockfile_BuildFlagsByMode(t *testing.T) { require.Zero(t, defaultResults.ExitCode) assert.Contains(t, defaultResults.Stdout, "--skip-lock-validation") - previewResults, err := cmdtest.NewScenarioTest( - "--without-lockfile", "component", "build", "--help", - ).Locally().Run(t) - require.NoError(t, err) - require.Zero(t, previewResults.ExitCode) - assert.NotContains(t, previewResults.Stdout, "--skip-lock-validation") } -// TestWithoutLockfile_LegacyCommandIsNoOp verifies that a lock-file command invoked -// in lock-file-free mode reports that it does nothing instead of failing. -func TestWithoutLockfile_LegacyCommandIsNoOp(t *testing.T) { - t.Parallel() - - if testing.Short() { - t.Skip("skipping long test") - } - - results, err := cmdtest.NewScenarioTest( - "--without-lockfile", "component", "update", "--all-components", - ).Locally().Run(t) - require.NoError(t, err) - require.Zero(t, results.ExitCode) - assert.Contains(t, results.Stdout+results.Stderr, "no longer does anything") -} - -// TestWithoutLockfile_AgentSkillsByMode verifies that the emitted agent skills -// describe the workflow of the selected mode. -func TestWithoutLockfile_AgentSkillsByMode(t *testing.T) { +func TestLocklessComponents_AgentSkillsCoverBothWorkflows(t *testing.T) { t.Parallel() if testing.Short() { @@ -148,34 +77,6 @@ func TestWithoutLockfile_AgentSkillsByMode(t *testing.T) { assert.Contains(t, defaultResults.Stdout, "azldev-update-component") assert.Contains(t, defaultResults.Stdout, "azldev-refresh-upstream-commit") - previewResults, err := cmdtest.NewScenarioTest( - "--without-lockfile", "docs", "agent", "show", - ).Locally().Run(t) - require.NoError(t, err) - require.Zero(t, previewResults.ExitCode) - assert.Contains(t, previewResults.Stdout, "azldev-refresh-upstream-commit") - assert.NotContains(t, previewResults.Stdout, "azldev-update-component") -} - -// TestWithoutLockfile_ComponentChanged compares components across two commits in -// lock-file-free mode, where change detection compares the project configuration -// resolved at each ref instead of stored lock files. -// -// Flow: -// 1. Create a project with two local components (curl, bash). -// 2. Commit the project as the baseline. -// 3. In a second commit, change only curl's spec content. -// 4. Run 'azldev --without-lockfile component changed' between the two commits. -// 5. Assert curl is "changed" (its spec directory contents differ) and bash is -// "unchanged", with no lock files anywhere in the project. -func TestWithoutLockfile_ComponentChanged(t *testing.T) { - t.Parallel() - - if testing.Short() { - t.Skip("skipping long test") - } - - testLocklessComponentChanged(t, true) } func TestLocklessComponents_ComponentChanged(t *testing.T) { @@ -185,12 +86,6 @@ func TestLocklessComponents_ComponentChanged(t *testing.T) { t.Skip("skipping long test") } - testLocklessComponentChanged(t, false) -} - -func testLocklessComponentChanged(t *testing.T, global bool) { - t.Helper() - azldevBin, err := testhelpers.FindTestBinary() require.NoError(t, err) @@ -228,10 +123,8 @@ func testLocklessComponentChanged(t *testing.T, global bool) { project.Serialize(t, projectDir) patchProjectForLocal(t, projectDir) - if !global { - require.NoError(t, os.WriteFile(filepath.Join(projectDir, "lockless-components.txt"), - []byte("curl\n"), fileperms.PublicFile)) - } + require.NoError(t, os.WriteFile(filepath.Join(projectDir, "lockless-components.txt"), + []byte("curl\n"), fileperms.PublicFile)) gitInDir(t, projectDir, "init") gitInDir(t, projectDir, "config", "user.email", "test@test.com") @@ -258,9 +151,6 @@ func testLocklessComponentChanged(t *testing.T, global bool) { "-C", projectDir, "--no-default-config", "component", "changed", "--from", fromRef, "-a", "--include-unchanged", "-q", "-O", "json", } - if global { - args = append([]string{"--without-lockfile"}, args...) - } cmd := exec.CommandContext(t.Context(), azldevBin, args...) out, err := cmd.CombinedOutput() @@ -282,19 +172,17 @@ func testLocklessComponentChanged(t *testing.T, global bool) { require.True(t, ok, "bash should be in results (--all-components)") assert.Equal(t, "unchanged", bashResult.ChangeType, "bash is untouched") - // No lock files are consulted or created in this mode. + // Change detection does not create lock files. entries, err := os.ReadDir(projectDir) require.NoError(t, err) for _, entry := range entries { - assert.NotEqual(t, "locks", entry.Name(), "lock-file-free mode must not create a lock directory") + assert.NotEqual(t, "locks", entry.Name(), "change detection must not create a lock directory") assert.False(t, strings.HasSuffix(entry.Name(), ".lock")) } } -// TestWithoutLockfile_MCPToolsByMode verifies that the MCP tool surface follows the -// selected mode's command set. -func TestWithoutLockfile_MCPToolsByMode(t *testing.T) { +func TestLocklessComponents_MCPTools(t *testing.T) { t.Parallel() if testing.Short() { @@ -309,10 +197,5 @@ func TestWithoutLockfile_MCPToolsByMode(t *testing.T) { require.Zero(t, defaultResults.ExitCode) assert.Contains(t, defaultResults.Stdout, `"component-history"`) - previewResults, err := cmdtest.NewScenarioTest("--without-lockfile", "advanced", "mcp"). - Locally().WithStdin(strings.NewReader(listToolsRequest)).Run(t) - require.NoError(t, err) - require.Zero(t, previewResults.ExitCode) - assert.NotContains(t, previewResults.Stdout, `"component-history"`) - assert.Contains(t, previewResults.Stdout, `"component-changed"`) + assert.Contains(t, defaultResults.Stdout, `"component-changed"`) } From b31ec107319e9f5920b661986fd2f833643c6117 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 3 Nov 2026 00:08:55 +0000 Subject: [PATCH 03/70] feat(component): implement lockless upstream render lifecycle Prepare upstream release and changelog state from committed render history before applying overlays and existing synthetic-history processing. Bootstrap autorelease changelogs, compare fully resolved TOML pins, collect upstream changes, and run rpmdev-bumpspec for every lockless upstream render. Publish the processed output without .git and commit each component directory separately, preserving unrelated staged changes. Keep local and lock-file rendering unchanged, and leave --check-only read-only. Update user and agent guidance and the generated CLI reference. Add in-memory regressions for history, release handling, ordering, and output-only commits. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3497f2b2-c7fb-456e-a58f-6889b6ec67e2 --- docs/user/how-to/preview-without-lockfile.md | 48 +- .../reference/cli/azldev_component_render.md | 10 + docs/user/reference/config/components.md | 8 +- .../app/azldev/agentskill/agentskill_test.go | 15 + .../azldev/agentskill/content/azldev.md.tmpl | 8 + .../agentskill/content/comp-toml.md.tmpl | 4 +- .../content/refresh-upstream-commit.md.tmpl | 50 +- .../content/withoutlockfile/azldev.md.tmpl | 6 + .../content/withoutlockfile/comp-toml.md.tmpl | 5 + .../cmds/component/changed_upstreamcommit.go | 70 ++- internal/app/azldev/cmds/component/render.go | 36 +- .../azldev/cmds/component/render_lockless.go | 525 ++++++++++++++++++ .../render_lockless_internal_test.go | 492 ++++++++++++++++ .../app/azldev/core/sources/mockprocessor.go | 20 +- .../azldev/core/sources/mockprocessor_test.go | 29 + .../app/azldev/core/sources/sourceprep.go | 33 +- .../azldev/core/sources/sourceprep_test.go | 38 ++ 17 files changed, 1330 insertions(+), 67 deletions(-) create mode 100644 internal/app/azldev/cmds/component/render_lockless.go create mode 100644 internal/app/azldev/cmds/component/render_lockless_internal_test.go diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index fd46ea5a0..0dde5b1db 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -102,12 +102,58 @@ resolved components are created or updated before the command exits with an error. Failed components remain unchanged, and orphan pruning is skipped for that run. -Commit the refreshed TOML together with the rendered output: synthetic dist-git +Commit the refreshed TOML before rendering: synthetic dist-git history — and therefore `%autorelease` and `%autochangelog` expansion — is derived from committed changes to the generated file. Unlike the default mode, there is no fingerprint to compare the working tree against, so uncommitted changes do not produce a synthetic commit. +## Render and Commit an Upstream Component + +The following process applies only to **upstream components selected as +lockless**. Local components and components using lock files retain their +existing rendering behavior. + +The renderer clones the selected upstream dist-git to temporary storage. It +finds the existing committed output under +`//`, ignoring +uncommitted spec changes. For an existing non-`%autorelease` upstream spec, it +carries over the committed rendered spec's exact `Release:` value and +`%changelog` section. + +For an initial `%autorelease` render, it generates the complete upstream +`changelog` file (replacing any existing file), calculates the numeric upstream +release, and adds or replaces the macro's `-b` parameter. These rpmautospec +operations run inside the shared mock chroot. + +The last commit touching the component output is the **last render commit**. +The renderer fully loads TOML, including includes and component defaults, at +that commit and at the current project commit to compare `spec.upstream-commit`. +Different pins, missing render history, or an error loading the old pin mean +upstream changed. It records `git log --oneline ..` from the +upstream checkout; without an old pin it records a one-line version update. +An error reading the current pin is fatal. + +When upstream changes exist and `Version:` differs from the committed rendered +version (or there is no previous output), it restores the original upstream +`Release:`. It then always invokes host `rpmdev-bumpspec -c` with +`updated ` followed by upstream changes on separate lines. +The tool's no-op for `%autorelease` specs is accepted. + +Existing overlays, synthetic history, RPM expansion, and sidecar filtering run +after these steps. The renderer replaces the component output, excluding `.git`, +and commits only that directory. The subject is `updated `; the +body lists upstream changes, if any. Unrelated staged files are not committed. +Unchanged Git content does not produce an empty commit. A commit failure leaves +the rendered files available for inspection. + +Install `rpmdevtools` on the host and configure Git author identity before +rendering. Output (including a custom `--output-dir`) must be inside the project +Git repository. `--check-only` prepares and compares temporary output but never +replaces rendered files or creates project commits. Because ordinary release +bumping is part of rendering, a new render may differ even without new upstream +commits. + ## Detect Changed Components ```bash diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 009afab51..2c3175753 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -26,6 +26,16 @@ its lock file, using configured upstream commits and their TOML history instead. All other components use lock files. A missing or empty file means all components use lock files. +For lockless upstream components, prepare release and changelog state from the +last committed render before applying the existing overlays and processing. +Run rpmdev-bumpspec with "updated " and any upstream changes. +After replacing the output, automatically commit that component directory to +the project repository, without including unrelated staged files. The output +must be inside that repository; Git author identity must be configured and +rpmdev-bumpspec must be installed. Commit updated upstream-commit TOML before +rendering. --check-only never replaces output or creates project commits. +Local components and components using lock files keep their existing behavior. + When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to any component in the project config). Per-component dirs that ARE in config diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index f289122b5..cb925c80d 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -142,7 +142,13 @@ or `azldev component prepare-sources` to preview transactional `rpmdev-bumpspec` handling instead. This temporary Azure Linux migration flag uses the existing fixed changelog metadata and requires `rpmdev-bumpspec`, `rpmdev-packager`, `rpm`, `rpmspec`, and Python with the RPM module only when -explicitly selected. `%autorelease` behavior is unchanged. +explicitly selected for this transactional strategy. `%autorelease` behavior is unchanged. + +**Exception:** Rendering a lockless upstream component always invokes host +`rpmdev-bumpspec -c` before the existing processing, without requiring the flag. +Its `%autorelease` no-op is accepted. See the +[lockless rendering workflow](../../how-to/preview-without-lockfile.md#render-and-commit-an-upstream-component) +for release/changelog carryover and automatic output commits. ## Render Configuration diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index ca143cd26..2c5aedbbc 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -170,6 +170,21 @@ func TestComponentTomlSkillDocumentsPerComponentRenderModes(t *testing.T) { } } +func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "host `rpmdev-bumpspec`") + assert.Contains(t, doc, "`--check-only` never creates project commits") + } + + doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "fully parsed committed TOML") + assert.Contains(t, doc, "automatically commits only that directory") + assert.NotContains(t, doc, "git commit --amend") +} + func TestImageSkillDocumentsSKUGroupGuidance(t *testing.T) { doc, err := agentskill.SkillDocument("azldev-image", testParams()) require.NoError(t, err) diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 8bddc7993..136316f01 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -38,6 +38,14 @@ images assemble RPMs into bootable artifacts. workflow per component for all commands. Listed components ignore locks and use generated upstream-commit TOML instead. All others use locks. +For listed upstream components, rendering carries release/changelog state from +the committed output, runs host `rpmdev-bumpspec`, then applies the existing +overlays, synthetic history, RPM expansion, and filtering. It replaces and +automatically commits only that component's output directory; unrelated staged +files are excluded. Commit refreshed TOML before rendering. Configure Git author +identity and install `rpmdevtools`. `--check-only` never creates project commits. +Local and locked components retain their existing render behavior. + Top-level commands in this build: {{ range .TopLevelCommands }}- `azldev {{ .Name }}` — {{ .Short }} diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 600b7886e..05d42ecb8 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -92,7 +92,9 @@ migrates. Only that path requires `rpmdev-bumpspec`, `rpmdev-packager`, `rpm`, identity/comment/datestamp and strict source-EVR transaction checks; locks and synthetic history still determine the number and order of bumps. Without the flag, no bumpspec runtime discovery occurs and the legacy static calculation is -used. `%autorelease` behavior is unchanged on either path. +used outside the lockless upstream render workflow. That workflow always runs +host `rpmdev-bumpspec -c` before existing processing, accepts its `%autorelease` +no-op, and automatically commits the component output. Commit input TOML first. ## Render configuration diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 62e26c2c3..51fdf7f38 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -44,37 +44,35 @@ Select this workflow by listing the component's exact name in `lockless-components.txt` beside the top-level `azldev.toml`. Refresh skips unlisted components and preserves their generated TOML files. -For edits that do not change source resolution, render and commit the result. -The generated upstream commit TOML does not change: +Commit component inputs before rendering. For upstream components, rendering +compares the fully parsed committed TOML at the last render commit and at HEAD; +uncommitted generated pins do not count as a committed upstream change. ```sh -azldev comp render -p git add \ - {{ .RenderedSpecsDir }}/// + {{ .UpstreamCommitsDir }}/.toml git commit -m "fix(): ..." +azldev comp render -p ``` -Synthetic dist-git history is derived from committed upstream-commit TOML changes. Changes -that do not refresh the upstream commit do not add a synthetic history entry or -trigger an automatic release bump. - -## Changing source resolution - -A source-resolution change follows the same rule, using one commit followed by a -post-render amend: - -1. Change the upstream distro/version or snapshot, then `azldev comp refresh-upstream-commit -p `; sanity-check `{{ .UpstreamCommitsDir }}/.toml`. -2. `azldev comp render -p ` — the spec body now tracks the newly resolved source. `%changelog` / `Release:` still reflect the previous source; that is expected until you commit. -3. Iterate on overlays / patches / build config as the new source requires, re-rendering after each change. Re-run `refresh-upstream-commit` only if you change a source-resolution input again. -4. Stage and commit all component inputs changed above with the refreshed TOML and rendered output: - - ```sh - git add \ - {{ .UpstreamCommitsDir }}/.toml \ - {{ .RenderedSpecsDir }}/// - git commit -m "feat(): refresh upstream source" - ``` -5. `azldev comp render -p ` — `%changelog` / `Release:` now reflect the new commit TOML. -6. `git add {{ .RenderedSpecsDir }}///`, then `git commit --amend --no-edit` so the source change and rendered output land together. +Rendering preserves the committed downstream Release and changelog for static +specs, resets Release to its original upstream value on a version change, and +runs host `rpmdev-bumpspec -c` with `updated ` plus upstream +changes. An initial `%autorelease` render generates the complete upstream +changelog and calculates its base release inside mock. Existing overlays, +synthetic history, RPM expansion, and filtering still run afterward. + +Rendering then replaces `{{ .RenderedSpecsDir }}///` and +automatically commits only that directory. The subject is +`updated ` and the body contains upstream changes, if any. +It never includes unrelated staged files or creates empty commits. Configure +Git author identity and install host `rpmdevtools`. `--check-only` never replaces +output or commits, but a new static release bump may cause drift without new +upstream changes. + +For source-resolution changes, run `refresh-upstream-commit` first and review the +generated pin, then follow the commit-before-render sequence above. Do not amend +the input commit to absorb the automatically generated render commit. Local +components retain their previous rendering and manual output-commit workflow. Generated by `azldev docs agent`; do not hand-edit. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index c1f87f153..6bf8cf159 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -33,6 +33,12 @@ images assemble RPMs into bootable artifacts. ## Common commands +For upstream components, rendering preserves committed downstream release and +changelog state before running host `rpmdev-bumpspec` and existing processing. +It replaces and automatically commits only the component output directory. +Commit refreshed TOML before rendering; configure Git author identity and install +`rpmdevtools`. `--check-only` never creates project commits. Local rendering is unchanged. + Top-level commands in this build: {{ range .TopLevelCommands }}- `azldev {{ .Name }}` — {{ .Short }} diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index aef27efce..6ddb3295e 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -86,6 +86,11 @@ calculation = "manual" ## Render configuration +Upstream rendering always runs host `rpmdev-bumpspec -c` before the existing +processing and automatically commits the component output directory. Commit +input TOML first, install host `rpmdevtools`, and configure Git author identity. +`--check-only` never creates project commits. Local rendering is unchanged. + azldev reads `lockless-components.txt` beside the top-level `azldev.toml`. Listed component names use the lockless workflow for every command; all others use lock files. A missing or empty list means all components use diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit.go b/internal/app/azldev/cmds/component/changed_upstreamcommit.go index 426a3f8f2..68f338c3b 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit.go @@ -130,6 +130,46 @@ func loadHistoricalProject( tree *object.Tree, projectRelDir string, ) (*historicalProject, error) { + historicalEnv, err := loadHistoricalEnv(env, tree, projectRelDir) + if err != nil { + return nil, err + } + + config := historicalEnv.Config() + resolver := components.NewResolver(historicalEnv) + + resolvedComponents, err := resolver.FindAllComponents() + if err != nil { + return nil, fmt.Errorf("resolving components:\n%w", err) + } + + result := &historicalProject{ + components: make(map[string]projectconfig.ComponentConfig, resolvedComponents.Len()), + comparisonInputs: make(map[string]componentComparisonInputs, resolvedComponents.Len()), + componentGroups: make(map[string][]string, len(config.ComponentGroups)), + } + + if err := populateHistoricalComponents( + result, historicalEnv.FS(), historicalEnv, resolvedComponents, env, + ); err != nil { + return nil, err + } + + if err := populateHistoricalGroups(result, resolver, config); err != nil { + return nil, err + } + + result.renderedSpecsRelDir, err = repoRelPath(snapshotRepoRoot, config.Project.RenderedSpecsDir) + if err != nil { + return nil, fmt.Errorf("resolving rendered specs directory:\n%w", err) + } + + return result, nil +} + +func loadHistoricalEnv( + env *azldev.Env, tree *object.Tree, projectRelDir string, +) (*azldev.Env, error) { snapshotFS := afero.NewMemMapFs() if err := copyTreeToFS(tree, snapshotFS, snapshotRepoRoot); err != nil { @@ -166,35 +206,7 @@ func loadHistoricalProject( }) historicalEnv.SetPermissiveConfigParsing(env.PermissiveConfigParsing()) - resolver := components.NewResolver(historicalEnv) - - resolvedComponents, err := resolver.FindAllComponents() - if err != nil { - return nil, fmt.Errorf("resolving components:\n%w", err) - } - - result := &historicalProject{ - components: make(map[string]projectconfig.ComponentConfig, resolvedComponents.Len()), - comparisonInputs: make(map[string]componentComparisonInputs, resolvedComponents.Len()), - componentGroups: make(map[string][]string, len(config.ComponentGroups)), - } - - if err := populateHistoricalComponents( - result, snapshotFS, historicalEnv, resolvedComponents, env, - ); err != nil { - return nil, err - } - - if err := populateHistoricalGroups(result, resolver, config); err != nil { - return nil, err - } - - result.renderedSpecsRelDir, err = repoRelPath(snapshotRepoRoot, config.Project.RenderedSpecsDir) - if err != nil { - return nil, fmt.Errorf("resolving rendered specs directory:\n%w", err) - } - - return result, nil + return historicalEnv, nil } func populateHistoricalGroups( diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 349ee2987..f8e9637cd 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -18,6 +18,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/dirdiff" @@ -74,6 +75,16 @@ its lock file, using configured upstream commits and their TOML history instead. All other components use lock files. A missing or empty file means all components use lock files. +For lockless upstream components, prepare release and changelog state from the +last committed render before applying the existing overlays and processing. +Run rpmdev-bumpspec with "updated " and any upstream changes. +After replacing the output, automatically commit that component directory to +the project repository, without including unrelated staged files. The output +must be inside that repository; Git author identity must be configured and +rpmdev-bumpspec must be installed. Commit updated upstream-commit TOML before +rendering. --check-only never replaces output or creates project commits. +Local components and components using lock files keep their existing behavior. + When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to any component in the project config). Per-component dirs that ARE in config @@ -159,10 +170,11 @@ const ( ) // RenderComponents renders the post-overlay spec and sidecar files for each -// selected component into the output directory. Processing is done in three phases: +// selected component into the output directory. Processing is done in four phases: // 1. Parallel source preparation (clone, overlay, synthetic git) // 2. Batch mock processing (rpmautospec + spectool in a single chroot call) // 3. Parallel finishing (filter files, remove .git, copy output) +// 4. Serial project commits for lockless upstream component output func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, error) { if err := resolveAndValidateOutputDir(env, options); err != nil { return nil, err @@ -256,6 +268,8 @@ func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, // matches the expected single-marker shape and flags drift on mismatch. writeFailureMarkers(env.FS(), results, options.Force, options.CheckOnly) + commitLocklessRenders(env, prepared, results, options.CheckOnly) + // Sort results alphabetically for consistent output. sortRenderResults(results) @@ -377,6 +391,7 @@ type preparedComponent struct { comp components.Component specFilename string // e.g., "curl.spec" compOutputDir string // validated output path computed in phase 1 + update *locklessRenderUpdate } // prepResult pairs a prepared component (on success) or a render result (on error). @@ -486,7 +501,8 @@ func prepareOneComponent( }} } - prep, err := prepareComponentSources(env, mockProcessor, comp, stagingDir, specEditor, rpmDevBumpspec, lockless) + prep, err := prepareComponentSources(env, mockProcessor, comp, stagingDir, compOutputDir, + specEditor, rpmDevBumpspec, lockless) if err != nil { slog.Error("Failed to prepare component sources", "component", componentName, "error", err) @@ -512,6 +528,7 @@ func prepareComponentSources( mockProcessor *sources.MockProcessor, comp components.Component, stagingDir string, + compOutputDir string, specEditor spec.EditorMode, rpmDevBumpspec bool, lockless bool, @@ -550,6 +567,20 @@ func prepareComponentSources( sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), sources.WithMockProcessor(mockProcessor), ) + + var update *locklessRenderUpdate + + if lockless && comp.GetConfig().Spec.SourceType == projectconfig.SpecSourceTypeUpstream { + preparerOpts = append(preparerOpts, sources.WithFetchedSources( + func(_ context.Context, fetched components.Component, dir string) error { + var prepErr error + //nolint:contextcheck // The callback context is the captured preparation env. + update, prepErr = prepareLocklessRender(env, mockProcessor, fetched, dir, compOutputDir, specEditor) + + return prepErr + })) + } + if rpmDevBumpspec { preparerOpts = append(preparerOpts, sources.WithRPMDevBumpspec(env, env.WorkDir(), "")) } @@ -575,6 +606,7 @@ func prepareComponentSources( return &preparedComponent{ comp: comp, specFilename: filepath.Base(specPath), + update: update, }, nil } diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go new file mode 100644 index 000000000..dc47c460f --- /dev/null +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -0,0 +1,525 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" +) + +type locklessRenderUpdate struct { + repoRoot string + path string + subject string + changes string +} + +type lastRender struct { + spec string + previous string + current string +} + +type renderAutorelease interface { + GenerateChangelog(ctx context.Context, dir, filename string) (string, error) + CalculateBaseRelease(ctx context.Context, dir, filename string) (string, error) +} + +func prepareLocklessRender( + env *azldev.Env, autospec renderAutorelease, comp components.Component, + dir, outputDir string, editor spec.EditorMode, +) (*locklessRenderUpdate, error) { + repo, root, err := openChangedRepo(env) + if err != nil { + return nil, err + } + + path, err := repoRelPath(root, outputDir) + if err != nil { + return nil, fmt.Errorf("lockless render output must be inside the project repository:\n%w", err) + } + + history, err := readLastRender(env, repo, root, path, comp.GetName()) + if err != nil { + return nil, fmt.Errorf("reading committed render state:\n%w", err) + } + + update := &locklessRenderUpdate{ + repoRoot: root, + path: path, + subject: "updated " + comp.GetName(), + } + if err := prepareLocklessSpec(env, autospec, dir, comp.GetName(), editor, history, update); err != nil { + return nil, err + } + + return update, nil +} + +func readLastRender( + env *azldev.Env, repo *gogit.Repository, root, path, name string, +) (lastRender, error) { + var history lastRender + + head, err := repo.Head() + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return history, nil + } + + if err != nil { + return history, fmt.Errorf("reading project HEAD:\n%w", err) + } + + tree, err := resolveTree(repo, head.Hash().String()) + if err != nil { + return history, err + } + + _, err = tree.Tree(path) + if errors.Is(err, object.ErrDirectoryNotFound) { + return history, nil + } + + if err != nil { + return history, fmt.Errorf("reading committed dist-git directory:\n%w", err) + } + + file, err := tree.File(filepath.ToSlash(filepath.Join(path, name+".spec"))) + if err != nil { + return history, fmt.Errorf("reading committed spec:\n%w", err) + } + + history.spec, err = file.Contents() + if err != nil { + return history, fmt.Errorf("reading committed spec contents:\n%w", err) + } + + log, err := repo.Log(&gogit.LogOptions{ + From: head.Hash(), + PathFilter: func(file string) bool { + return strings.HasPrefix(file, path+"/") + }, + }) + if err != nil { + return history, fmt.Errorf("finding last render commit:\n%w", err) + } + defer log.Close() + + last, err := log.Next() + if errors.Is(err, io.EOF) { + return history, nil + } + + if err != nil { + return history, fmt.Errorf("reading last render commit:\n%w", err) + } + + history.previous, history.current, err = renderedUpstreamCommits(env, root, name, tree, last) + + return history, err +} + +func renderedUpstreamCommits( + env *azldev.Env, root, name string, tree *object.Tree, last *object.Commit, +) (previous string, current string, err error) { + projectPath, err := repoRelPath(root, env.ProjectDir()) + if err != nil { + return "", "", err + } + + current, err = renderUpstreamCommit(env, tree, projectPath, name) + if err != nil { + return "", "", fmt.Errorf("reading current render upstream commit:\n%w", err) + } + + lastTree, err := last.Tree() + if err == nil { + previous, err = renderUpstreamCommit(env, lastTree, projectPath, name) + } + + if err != nil { + slog.Warn("Cannot read last render upstream commit; treating upstream as changed", + "component", name, "error", err) + + previous = "" + } + + return previous, current, nil +} + +func renderUpstreamCommit(env *azldev.Env, tree *object.Tree, projectPath, name string) (string, error) { + historicalEnv, err := loadHistoricalEnv(env, tree, projectPath) + if err != nil { + return "", err + } + + comp, err := components.NewResolver(historicalEnv).GetComponentByName(name) + if err != nil { + return "", fmt.Errorf("resolving historical component %#q:\n%w", name, err) + } + + commit := comp.GetConfig().Spec.UpstreamCommit + if commit == "" { + return "", fmt.Errorf("component %#q has no configured 'spec.upstream-commit'", name) + } + + return commit, nil +} + +func prepareLocklessSpec( + env *azldev.Env, autospec renderAutorelease, dir, name string, editor spec.EditorMode, + history lastRender, update *locklessRenderUpdate, +) error { + path, err := findSpecFile(env.FS(), dir, name) + if err != nil { + return err + } + + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return fmt.Errorf("reading upstream spec:\n%w", err) + } + + opened, err := spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor)) + if err != nil { + return fmt.Errorf("parsing upstream spec:\n%w", err) + } + + original, err := opened.GetLastTag("", "Release") + if err != nil { + return fmt.Errorf("reading upstream Release:\n%w", err) + } + + version, err := opened.GetLastTag("", "Version") + if err != nil { + return fmt.Errorf("reading upstream Version:\n%w", err) + } + + if err := collectRenderUpstreamChanges(env, dir, name, version, history, update); err != nil { + return err + } + + release, err := locklessRenderRelease(env, autospec, dir, filepath.Base(path), original, history) + if err != nil { + return err + } + + release, err = restoreLocklessRelease(original, release, version, history, update.changes != "") + if err != nil { + return err + } + + if err := opened.SetTag("", "Release", release); err != nil { + return fmt.Errorf("updating upstream Release:\n%w", err) + } + + resultBytes, err := locklessSpecBytes(opened, history.spec, original, editor) + if err != nil { + return err + } + + if err := fileutils.WriteFile(env.FS(), path, resultBytes, fileperms.PublicFile); err != nil { + return fmt.Errorf("writing prepared upstream spec:\n%w", err) + } + + entry := update.subject + if update.changes != "" { + entry += "\n" + update.changes + } + + return bumpLocklessSpec(env, path, entry) +} + +func locklessSpecBytes(opened *spec.Spec, previous, release string, editor spec.EditorMode) ([]byte, error) { + var result bytes.Buffer + if err := opened.Serialize(&result); err != nil { + return nil, fmt.Errorf("serializing upstream spec:\n%w", err) + } + + if previous != "" && !sources.ReleaseUsesAutorelease(release) { + return restoreRenderChangelog(result.Bytes(), []byte(previous), editor) + } + + return result.Bytes(), nil +} + +func collectRenderUpstreamChanges( + env *azldev.Env, dir, name, version string, history lastRender, update *locklessRenderUpdate, +) error { + if history.previous == "" { + update.changes = fmt.Sprintf("Update %s to version %s", name, version) + + return nil + } + + if history.previous == history.current { + return nil + } + + changes, err := git.RunInDir(env, env, dir, + "log", "--oneline", "--end-of-options", history.previous+".."+history.current, "--") + if err != nil { + return fmt.Errorf("collecting upstream changes:\n%w", err) + } + + update.changes = changes + + return nil +} + +func locklessRenderRelease( + env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, +) (string, error) { + if history.spec != "" || !sources.ReleaseUsesAutorelease(release) || env.DryRun() { + return release, nil + } + + changelog, err := autospec.GenerateChangelog(env, dir, filename) + if err != nil { + return "", fmt.Errorf("generating initial upstream changelog:\n%w", err) + } + + if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "changelog"), []byte(changelog), + fileperms.PublicFile); err != nil { + return "", fmt.Errorf("writing initial upstream changelog:\n%w", err) + } + + calculated, err := autospec.CalculateBaseRelease(env, dir, filename) + if err != nil { + return "", fmt.Errorf("calculating upstream release base:\n%w", err) + } + + return withAutoreleaseBase(release, calculated) +} + +func restoreLocklessRelease( + original, release, version string, history lastRender, upstreamChanged bool, +) (string, error) { + if history.spec == "" && upstreamChanged { + return original, nil + } + + if history.spec == "" { + return release, nil + } + + previous, err := spec.OpenSpec(strings.NewReader(history.spec)) + if err != nil { + return "", fmt.Errorf("parsing committed dist-git spec:\n%w", err) + } + + if !sources.ReleaseUsesAutorelease(original) { + release, err = previous.GetLastTag("", "Release") + if err != nil { + return "", fmt.Errorf("reading committed Release:\n%w", err) + } + } + + if !upstreamChanged { + return release, nil + } + + oldVersion, err := previous.GetLastTag("", "Version") + if err != nil { + return "", fmt.Errorf("reading committed Version:\n%w", err) + } + + if oldVersion != version { + return original, nil + } + + return release, nil +} + +var autoreleaseBaseArg = regexp.MustCompile(`(^|\s)-b(?:\s+|=)?[0-9]+`) + +func withAutoreleaseBase(release, output string) (string, error) { + fields := strings.Fields(output) + if len(fields) == 0 { + return "", errors.New("rpmautospec returned an empty release number") + } + + number := fields[len(fields)-1] + + value, err := strconv.Atoi(number) + if err != nil || value < 1 { + return "", fmt.Errorf("rpmautospec returned invalid release number %#q", output) + } + + if autoreleaseBaseArg.MatchString(release) { + return autoreleaseBaseArg.ReplaceAllString(release, "${1}-b "+number), nil + } + + if strings.HasPrefix(release, "%{autorelease") { + depth := 0 + + for index, char := range release { + switch char { + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return release[:index] + " -b " + number + release[index:], nil + } + } + } + } + + return release + " -b " + number, nil +} + +func restoreRenderChangelog(upstream, previous []byte, editor spec.EditorMode) ([]byte, error) { + oldStart, oldEnd, err := renderChangelogBounds(previous, editor) + if err != nil { + return nil, err + } + + start, end, err := renderChangelogBounds(upstream, editor) + if err != nil { + return nil, err + } + + result := append([]byte(nil), upstream[:start]...) + if start == len(upstream) && oldStart != oldEnd && !bytes.HasSuffix(result, []byte("\n")) { + result = append(result, '\n') + } + + result = append(result, previous[oldStart:oldEnd]...) + result = append(result, upstream[end:]...) + + return result, nil +} + +func renderChangelogBounds(content []byte, editor spec.EditorMode) (start int, end int, err error) { + opened, err := spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor)) + if err != nil { + return 0, 0, fmt.Errorf("parsing spec changelog:\n%w", err) + } + + lines := bytes.SplitAfter(content, []byte("\n")) + offset := func(line int) int { + size := 0 + for _, text := range lines[:min(line, len(lines))] { + size += len(text) + } + + return size + } + start, end = len(content), len(content) + + err = opened.Visit(func(ctx *spec.Context) error { + if ctx.Target.Section == nil || ctx.Target.Section.SectName != "%changelog" { + return nil + } + + switch ctx.Target.TargetType { + case spec.SectionStartTarget: + start = offset(ctx.CurrentLineNum) + case spec.SectionEndTarget: + end = offset(ctx.CurrentLineNum) + case spec.SpecStartTarget, spec.SectionLineTarget, spec.SpecEndTarget: + } + + return nil + }) + if err != nil { + return 0, 0, fmt.Errorf("finding spec changelog:\n%w", err) + } + + return start, end, nil +} + +func bumpLocklessSpec(env *azldev.Env, path, entry string) error { + if err := prereqs.RequireExecutable(env, sources.RPMDevBumpspecBinary, nil); err != nil { + return fmt.Errorf("lockless upstream rendering requires 'rpmdev-bumpspec'; install 'rpmdevtools':\n%w", err) + } + + raw := exec.CommandContext(env, sources.RPMDevBumpspecBinary, "-c", entry, path) + raw.Dir = filepath.Dir(path) + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + command, err := env.Command(raw) + if err != nil { + return fmt.Errorf("creating render release bump command:\n%w", err) + } + + if err := command.Run(env); err != nil { + return fmt.Errorf("bumping render release (%s):\n%w", stderr.String(), err) + } + + return nil +} + +func commitLocklessRenders( + env *azldev.Env, prepared []*preparedComponent, results []*RenderResult, checkOnly bool, +) { + if checkOnly || env.DryRun() { + return + } + + // All project index writes are serialized; source preparation and filtering + // remain parallel. A failed commit leaves rendered output intact for recovery. + for _, prep := range prepared { + if prep.update == nil || results[prep.index].Status != renderStatusOK { + continue + } + + if err := commitLocklessRender(env, prep.update); err != nil { + results[prep.index].Status = renderStatusError + results[prep.index].Error = err.Error() + } + } +} + +func commitLocklessRender(env *azldev.Env, update *locklessRenderUpdate) error { + if _, err := git.RunInDir(env, env, update.repoRoot, "add", "--all", "--", update.path); err != nil { + return fmt.Errorf("staging rendered component:\n%w", err) + } + + diff, err := git.RunInDir(env, env, update.repoRoot, "diff", "--cached", "--name-only", "--", update.path) + if err != nil { + return fmt.Errorf("checking staged render:\n%w", err) + } + + if diff == "" { + return nil + } + + args := []string{"commit", "--only", "-m", update.subject} + if update.changes != "" { + args = append(args, "-m", update.changes) + } + + args = append(args, "--", update.path) + if _, err := git.RunInDir(env, env, update.repoRoot, args...); err != nil { + return fmt.Errorf("committing rendered component:\n%w", err) + } + + return nil +} diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go new file mode 100644 index 000000000..6279ebc27 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -0,0 +1,492 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "context" + "errors" + "fmt" + "os/exec" + "strconv" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func renderTestSpec(version, release, changelog string) string { + return fmt.Sprintf("Name: curl\nVersion: %s\nRelease: %s\nSummary: Test\nLicense: MIT\n"+ + "\n%%description\nTest\n\n%%changelog\n%s", version, release, changelog) +} + +func TestReadLastRender_CommittedConfigAndSpec(t *testing.T) { + root := []byte(`includes = ["component.toml", "pins/*.toml"] +[project] +default-distro = { name = "test", version = "1" } +rendered-specs-dir = "SPECS" +[distros.test.versions."1"] +release-ver = "1" +`) + oldSpec := renderTestSpec("1", "9%{?dist}", "* Previous\n- Keep exactly\n") + repo, _ := testRepoWithCommits(t, []testRepoCommit{ + {files: map[string][]byte{ + "azldev.toml": root, + "component.toml": []byte("[components.curl.spec]\ntype = 'upstream'\n"), + "pins/curl.toml": []byte("[components.curl.spec]\nupstream-commit = 'aaaaaaa'\n"), + "SPECS/c/curl/curl.spec": []byte(oldSpec), + }}, + {files: map[string][]byte{ + "pins/curl.toml": []byte("[components.curl.spec]\nupstream-commit = 'bbbbbbb'\n"), + }}, + {files: map[string][]byte{"README": []byte("unrelated later commit")}}, + }) + worktree, err := repo.Worktree() + require.NoError(t, err) + dirty, err := worktree.Filesystem.Create("SPECS/c/curl/curl.spec") + require.NoError(t, err) + _, err = dirty.Write([]byte("uncommitted spec must not be used")) + require.NoError(t, err) + require.NoError(t, dirty.Close()) + + env := testutils.NewTestEnv(t) + history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl") + require.NoError(t, err) + assert.Equal(t, oldSpec, history.spec) + assert.Equal(t, "aaaaaaa", history.previous) + assert.Equal(t, "bbbbbbb", history.current) +} + +func TestReadLastRender_AbsentAndInvalidHistory(t *testing.T) { + valid := []byte(`[project] +default-distro = { name = "test", version = "1" } +[distros.test.versions."1"] +release-ver = "1" +[components.curl.spec] +type = "upstream" +upstream-commit = "bbbbbbb" +`) + for _, testCase := range []struct { + name string + oldConfig []byte + oldSpec bool + current []byte + wantCurrent string + wantError string + }{ + {name: "no committed directory", oldConfig: valid}, + { + name: "invalid previous config", oldSpec: true, oldConfig: []byte("invalid = ["), + current: valid, wantCurrent: "bbbbbbb", + }, + { + name: "invalid current config", oldSpec: true, oldConfig: valid, + current: []byte("invalid = ["), wantError: "current render upstream commit", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + files := map[string][]byte{"azldev.toml": testCase.oldConfig} + if testCase.oldSpec { + files["SPECS/c/curl/curl.spec"] = []byte(renderTestSpec("1", "1", "old\n")) + } + + commits := []testRepoCommit{{files: files}} + if testCase.current != nil { + commits = append(commits, testRepoCommit{files: map[string][]byte{"azldev.toml": testCase.current}}) + } + + repo, _ := testRepoWithCommits(t, commits) + env := testutils.NewTestEnv(t) + + history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl") + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + + return + } + + require.NoError(t, err) + assert.Empty(t, history.previous) + assert.Equal(t, testCase.wantCurrent, history.current) + }) + } +} + +type renderAutospecStub struct { + calls []string + err error +} + +func (s *renderAutospecStub) GenerateChangelog(_ context.Context, _, _ string) (string, error) { + s.calls = append(s.calls, "changelog") + + return "* Full upstream history\n- Original change\n", s.err +} + +func (s *renderAutospecStub) CalculateBaseRelease(_ context.Context, _, _ string) (string, error) { + s.calls = append(s.calls, "release") + + return "Calculated release number: 42\n", s.err +} + +func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { + const oldChangelog = "* Previous downstream entry\n- Preserve spaces \n" + for _, testCase := range []struct { + name string + version string + release string + history lastRender + wantRelease string + wantChanges string + bootstrap bool + }{ + { + name: "same upstream", version: "1", release: "1%{?dist}", + history: lastRender{ + spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "old", + }, + wantRelease: "9%{?dist}", + }, + { + name: "upstream changes without version change", version: "1", release: "1%{?dist}", + history: lastRender{ + spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "new", + }, + wantRelease: "9%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", + }, + { + name: "version change resets release", version: "2", release: "1%{?dist}", + history: lastRender{ + spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "new", + }, + wantRelease: "1%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", + }, + { + name: "initial static spec", version: "2", release: "3%{?dist}", + wantRelease: "3%{?dist}", wantChanges: "Update curl to version 2", + }, + { + name: "initial autorelease", version: "2", release: "%autorelease -b 7", + wantRelease: "%autorelease -b 7", wantChanges: "Update curl to version 2", bootstrap: true, + }, + { + name: "existing autorelease", version: "1", release: "%autorelease", + history: lastRender{ + spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "old", + }, + wantRelease: "%autorelease", + }, + { + name: "unknown previous upstream", version: "1", release: "1", + history: lastRender{spec: renderTestSpec("1", "9", oldChangelog), current: "new"}, + wantRelease: "9", wantChanges: "Update curl to version 1", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + const path = "/working/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, path, + []byte(renderTestSpec(testCase.version, testCase.release, "* Upstream\n- New entry\n")), + fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", []byte("stale"), + fileperms.PublicFile)) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{ + "git", "-C", "/working", "log", "--oneline", "--end-of-options", + "old..new", "--", + }, cmd.Args) + + return "abc123 Upstream fix\ndef456 Another fix\n", nil + } + called := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + called = true + + entry := "updated curl" + if testCase.wantChanges != "" { + entry += "\n" + testCase.wantChanges + } + + assert.Equal(t, []string{"rpmdev-bumpspec", "-c", entry, path}, cmd.Args) + prepared, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Contains(t, string(prepared), "Release: "+testCase.wantRelease+"\n") + + if testCase.history.spec != "" && testCase.release != "%autorelease" { + assert.True(t, strings.HasSuffix(string(prepared), oldChangelog)) + } else { + assert.True(t, strings.HasSuffix(string(prepared), "* Upstream\n- New entry\n")) + } + + return nil + } + stub := &renderAutospecStub{} + update := &locklessRenderUpdate{subject: "updated curl"} + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, + testCase.history, update) + require.NoError(t, err) + assert.True(t, called) + assert.Equal(t, testCase.wantChanges, update.changes) + + if testCase.bootstrap { + assert.Equal(t, []string{"changelog", "release"}, stub.calls) + + changelog, err := fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, "* Full upstream history\n- Original change\n", string(changelog)) + } else { + assert.Empty(t, stub.calls) + } + }) + } +} + +func TestWithAutoreleaseBase(t *testing.T) { + for _, testCase := range []struct{ release, want string }{ + {"%autorelease", "%autorelease -b 42"}, + {"%autorelease -b 7 -e test", "%autorelease -b 42 -e test"}, + {"%autorelease -b7", "%autorelease -b 42"}, + {"%{autorelease}", "%{autorelease -b 42}"}, + {"%{autorelease -b 7}", "%{autorelease -b 42}"}, + {"%autorelease -e %{suffix}", "%autorelease -e %{suffix} -b 42"}, + {"%{autorelease}%{?dist}", "%{autorelease -b 42}%{?dist}"}, + {"%{autorelease -e %{suffix}}", "%{autorelease -e %{suffix} -b 42}"}, + } { + got, err := withAutoreleaseBase(testCase.release, "Calculated release number: 42\n") + require.NoError(t, err) + assert.Equal(t, testCase.want, got) + } + + for _, output := range []string{"", "release: invalid", "0", "7.fc43"} { + _, err := withAutoreleaseBase("%autorelease", output) + require.Error(t, err) + } + + got, err := withAutoreleaseBase("%autorelease", "1") + require.NoError(t, err) + assert.Equal(t, "%autorelease -b 1", got) +} + +func TestPrepareComponentSources_LocalAndLockedSkipNewLifecycle(t *testing.T) { + for _, testCase := range []struct { + name string + source projectconfig.SpecSourceType + lockless bool + }{ + {name: "locked local", source: projectconfig.SpecSourceTypeLocal}, + {name: "lockless local", source: projectconfig.SpecSourceTypeLocal, lockless: true}, + {name: "locked upstream", source: projectconfig.SpecSourceTypeUpstream}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + ctrl := gomock.NewController(t) + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: testCase.source}, + }) + + manager := sourceproviders_test.NewMockSourceManager(ctrl) + manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).Return(nil) + + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + t.Cleanup(func() { newSourcePreparer = original }) + + // A successful fetch without a spec deliberately stops at overlays. + // The new lifecycle would instead require a project Git repository. + _, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/rendered", + spec.EditorLegacy, false, testCase.lockless) + require.ErrorContains(t, err, "failed to apply overlays") + }) + } +} + +func TestPrepareLocklessSpec_StopsOnAutospecFailure(t *testing.T) { + env := testutils.NewTestEnv(t) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(renderTestSpec("1", "%autorelease", "%autochangelog\n")), fileperms.PublicFile)) + + stub := &renderAutospecStub{err: errors.New("autospec failed")} + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, lastRender{}, + &locklessRenderUpdate{subject: "updated curl"}) + require.ErrorContains(t, err, "autospec failed") + assert.Equal(t, []string{"changelog"}, stub.calls) +} + +func TestBumpLocklessSpec_Failures(t *testing.T) { + for _, available := range []bool{false, true} { + t.Run(strconv.FormatBool(available), func(t *testing.T) { + env := testutils.NewTestEnv(t) + if available { + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + } + + called := false + env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + called = true + + return errors.New("invalid spec") + } + err := bumpLocklessSpec(env.Env, "/working/curl.spec", "updated curl") + + assert.Equal(t, available, called) + + if available { + require.ErrorContains(t, err, "invalid spec") + } else { + require.ErrorContains(t, err, "install 'rpmdevtools'") + } + }) + } +} + +func TestCommitLocklessRenders_CheckOnlyAndFailure(t *testing.T) { + for _, checkOnly := range []bool{false, true} { + t.Run(strconv.FormatBool(checkOnly), func(t *testing.T) { + env := testutils.NewTestEnv(t) + + const path = "/project/SPECS/c/curl/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte("rendered output"), fileperms.PublicFile)) + + called := false + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + called = true + + return "", errors.New("git identity is missing") + } + prepared := []*preparedComponent{{ + update: &locklessRenderUpdate{repoRoot: "/project", path: "SPECS/c/curl"}, + }} + results := []*RenderResult{{Status: renderStatusOK}} + commitLocklessRenders(env.Env, prepared, results, checkOnly) + assert.Equal(t, !checkOnly, called) + + if checkOnly { + assert.Equal(t, renderStatusOK, results[0].Status) + } else { + assert.Equal(t, renderStatusError, results[0].Status) + assert.Contains(t, results[0].Error, "git identity is missing") + } + + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Equal(t, "rendered output", string(content)) + }) + } +} + +func TestRestoreRenderChangelog_ExactBytesAndOtherSections(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + previous := []byte(renderTestSpec("1", "9", "* Old entry\n- Whitespace \n\n")) + upstream := []byte(renderTestSpec("2", "1", "* New entry\n- Discard\n") + "%files\n/file\n") + got, err := restoreRenderChangelog(upstream, previous, editor) + require.NoError(t, err) + assert.Contains(t, string(got), "%changelog\n* Old entry\n- Whitespace \n\n%files\n/file\n") + assert.NotContains(t, string(got), "Discard") + }) + } +} + +func TestRestoreRenderChangelog_MissingSection(t *testing.T) { + const oldChangelog = "%changelog\n* Old entry\n- Keep\n" + for _, testCase := range []struct{ upstream, previous, want string }{ + {"Name: curl", oldChangelog, "Name: curl\n" + oldChangelog}, + {"Name: curl\n", oldChangelog, "Name: curl\n" + oldChangelog}, + {"Name: curl", "Name: old\n", "Name: curl"}, + {"Name: curl\n%changelog\nDiscard\n", "Name: old\n", "Name: curl\n"}, + } { + got, err := restoreRenderChangelog([]byte(testCase.upstream), []byte(testCase.previous), spec.EditorLegacy) + require.NoError(t, err) + assert.Equal(t, testCase.want, string(got)) + } +} + +func TestCommitLocklessRenders_SkipsFailedComponents(t *testing.T) { + env := testutils.NewTestEnv(t) + + var commands [][]string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + commands = append(commands, cmd.Args) + assert.Equal(t, "SPECS/v/vim", cmd.Args[len(cmd.Args)-1]) + + if cmd.Args[3] == "diff" { + return "SPECS/v/vim/vim.spec\n", nil + } + + return "", nil + } + prepared := []*preparedComponent{ + {index: 0}, + {index: 1, update: &locklessRenderUpdate{repoRoot: "/project", path: "SPECS/c/curl"}}, + {index: 2, update: &locklessRenderUpdate{ + repoRoot: "/project", path: "SPECS/v/vim", subject: "updated vim", + }}, + } + results := []*RenderResult{ + {Status: renderStatusOK}, + {Status: renderStatusError, Error: "render failed"}, + {Status: renderStatusOK}, + } + commitLocklessRenders(env.Env, prepared, results, false) + assert.Equal(t, "render failed", results[1].Error) + assert.Equal(t, renderStatusOK, results[2].Status) + require.Len(t, commands, 3) + assert.Equal(t, "commit", commands[2][3]) +} + +func TestCommitLocklessRender_PathOnlyAndNoEmptyCommits(t *testing.T) { + for _, changed := range []bool{false, true} { + t.Run(strconv.FormatBool(changed), func(t *testing.T) { + env := testutils.NewTestEnv(t) + + var commands [][]string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + commands = append(commands, cmd.Args) + if cmd.Args[3] == "diff" && changed { + return "SPECS/c/curl/curl.spec\n", nil + } + + return "", nil + } + update := &locklessRenderUpdate{ + repoRoot: "/project", path: "SPECS/c/curl", subject: "updated curl", changes: "abc123 Fix", + } + require.NoError(t, commitLocklessRender(env.Env, update)) + assert.Equal(t, []string{"git", "-C", "/project", "add", "--all", "--", update.path}, commands[0]) + + if changed { + require.Len(t, commands, 3) + assert.Equal(t, []string{ + "git", "-C", "/project", "commit", "--only", "-m", "updated curl", + "-m", "abc123 Fix", "--", "SPECS/c/curl", + }, commands[2]) + } else { + assert.Len(t, commands, 2) + } + }) + } +} diff --git a/internal/app/azldev/core/sources/mockprocessor.go b/internal/app/azldev/core/sources/mockprocessor.go index a10ff7107..a1d8410a8 100644 --- a/internal/app/azldev/core/sources/mockprocessor.go +++ b/internal/app/azldev/core/sources/mockprocessor.go @@ -362,6 +362,22 @@ func (p *MockProcessor) runBatchScript( // the caller parses and validates it. The chroot is lazily initialized on first // use, shared with [MockProcessor.BatchProcess]. func (p *MockProcessor) CalculateRelease(ctx context.Context, specHostDir, specFilename string) (string, error) { + return p.runRPMAutospec(ctx, specHostDir, specFilename, "calculate-release", "--complete-release") +} + +// CalculateBaseRelease returns the numeric upstream release for an autorelease base. +func (p *MockProcessor) CalculateBaseRelease(ctx context.Context, specHostDir, specFilename string) (string, error) { + return p.runRPMAutospec(ctx, specHostDir, specFilename, "calculate-release", "--number-only") +} + +// GenerateChangelog returns the full changelog from the checkout's upstream history. +func (p *MockProcessor) GenerateChangelog(ctx context.Context, specHostDir, specFilename string) (string, error) { + return p.runRPMAutospec(ctx, specHostDir, specFilename, "generate-changelog") +} + +func (p *MockProcessor) runRPMAutospec( + ctx context.Context, specHostDir, specFilename string, args ...string, +) (string, error) { p.mu.Lock() defer p.mu.Unlock() @@ -383,7 +399,7 @@ func (p *MockProcessor) CalculateRelease(ctx context.Context, specHostDir, specF // on the bind-mounted .git, mirroring the render batch path. Wrapped in // `sh -c` so the shell operators survive shellquote joining in CmdInChroot. shellCmd := "git config --global --add safe.directory '*' && " + - "rpmautospec calculate-release --complete-release " + shellquote.Join(specInChroot) + shellquote.Join(append(append([]string{"rpmautospec"}, args...), specInChroot)...) cmd, err := runner.CmdInChroot(ctx, []string{"sh", "-c", shellCmd}, false) if err != nil { @@ -392,7 +408,7 @@ func (p *MockProcessor) CalculateRelease(ctx context.Context, specHostDir, specF out, err := cmd.RunAndGetOutput(ctx) if err != nil { - return "", fmt.Errorf("rpmautospec calculate-release failed in mock chroot:\n%w", err) + return "", fmt.Errorf("rpmautospec %s failed in mock chroot:\n%w", args[0], err) } return out, nil diff --git a/internal/app/azldev/core/sources/mockprocessor_test.go b/internal/app/azldev/core/sources/mockprocessor_test.go index c378a7e3f..a38debcf6 100644 --- a/internal/app/azldev/core/sources/mockprocessor_test.go +++ b/internal/app/azldev/core/sources/mockprocessor_test.go @@ -231,6 +231,35 @@ func TestCalculateRelease_Success(t *testing.T) { "git safe.directory must be configured before rpmautospec reads the .git history") } +func TestRenderRPMAutospecCommands(t *testing.T) { + for _, operation := range []string{"generate-changelog", "calculate-release --number-only"} { + t.Run(operation, func(t *testing.T) { + ctx := newMockProcessorTestCtx() + ctx.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Contains(t, strings.Join(cmd.Args, " "), + "rpmautospec "+operation+" /tmp/provenance/curl.spec") + + return "unchanged output\n", nil + } + processor := NewMockProcessor(ctx, "/mock/config") + + var ( + out string + err error + ) + + if operation == "generate-changelog" { + out, err = processor.GenerateChangelog(ctx, "/working", "curl.spec") + } else { + out, err = processor.CalculateBaseRelease(ctx, "/working", "curl.spec") + } + + require.NoError(t, err) + assert.Equal(t, "unchanged output\n", out) + }) + } +} + func TestCalculateRelease_InitFailurePropagates(t *testing.T) { t.Parallel() diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index ca6df85b3..e078232c1 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -140,6 +140,14 @@ func WithSkipLookaside() PreparerOption { } } +// WithFetchedSources runs prepare on the fetched checkout before overlays and +// synthetic history are applied. +func WithFetchedSources(prepare func(context.Context, components.Component, string) error) PreparerOption { + return func(p *sourcePreparerImpl) { + p.prepareFetchedSources = prepare + } +} + // WithUpstreamProvenance returns a [PreparerOption] that enables emission of // the %fedora_upstream_version and %fedora_upstream_release macros for Fedora // upstream components. distTag is the resolved %{?dist} expansion (e.g. @@ -184,11 +192,12 @@ func WithAllowNoHashes() PreparerOption { // Standard implementation of the [SourcePreparer] interface. type sourcePreparerImpl struct { - sourceManager sourceproviders.SourceManager - specEditor spec.EditorMode - fs opctx.FS - eventListener opctx.EventListener - dryRunnable opctx.DryRunnable + sourceManager sourceproviders.SourceManager + prepareFetchedSources func(context.Context, components.Component, string) error + specEditor spec.EditorMode + fs opctx.FS + eventListener opctx.EventListener + dryRunnable opctx.DryRunnable // withGitRepo, when true, enables dist-git creation by preserving the // upstream .git directory and generating synthetic commit history. @@ -322,6 +331,10 @@ func (p *sourcePreparerImpl) PrepareSources( component.GetName(), err) } + if err := p.prepareFetched(ctx, component, outputDir); err != nil { + return fmt.Errorf("preparing fetched sources for %#q:\n%w", component.GetName(), err) + } + // Fetch custom and downloaded source files. if !p.skipLookaside { if err := p.sourceManager.FetchFiles(ctx, component, outputDir); err != nil { @@ -360,6 +373,16 @@ func (p *sourcePreparerImpl) PrepareSources( return nil } +func (p *sourcePreparerImpl) prepareFetched( + ctx context.Context, component components.Component, dir string, +) error { + if p.prepareFetchedSources == nil { + return nil + } + + return p.prepareFetchedSources(ctx, component, dir) +} + func (p *sourcePreparerImpl) validateArchiveOverlayConfig(component components.Component) error { config := component.GetConfig() if err := config.ValidateArchiveOverlays(p.allowNoHashes); err != nil { diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index 8276689c3..97bc67129 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -4,12 +4,14 @@ package sources_test import ( + "context" "errors" "os" "path/filepath" "strings" "testing" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" @@ -27,6 +29,42 @@ import ( const testOutputDir = "/output" +func TestPrepareSources_FetchedSourcesRunBeforeOverlays(t *testing.T) { + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + manager := sourceproviders_test.NewMockSourceManager(ctrl) + ctx := testctx.NewCtx() + + component.EXPECT().GetName().AnyTimes().Return("curl") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Overlays: []projectconfig.ComponentOverlay{{ + Type: projectconfig.ComponentOverlayAppendSpecLines, + Lines: []string{"# overlay runs last"}, + }}, + }) + manager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir, gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile(ctx.FS(), filepath.Join(dir, "curl.spec"), + []byte("Name: curl\nRelease: 1\n"), fileperms.PublicFile) + }) + preparer, err := sources.NewPreparer(manager, ctx.FS(), ctx, ctx, + sources.WithSkipLookaside(), + sources.WithFetchedSources(func(_ context.Context, _ components.Component, dir string) error { + path := filepath.Join(dir, "curl.spec") + content, err := fileutils.ReadFile(ctx.FS(), path) + require.NoError(t, err) + assert.NotContains(t, string(content), "overlay runs last") + + return fileutils.WriteFile(ctx.FS(), path, append(content, []byte("# release prepared\n")...), + fileperms.PublicFile) + })) + require.NoError(t, err) + require.NoError(t, preparer.PrepareSources(ctx, component, testOutputDir, true)) + content, err := fileutils.ReadFile(ctx.FS(), testOutputDir+"/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "# release prepared\n# overlay runs last") +} + func TestNewPreparer(t *testing.T) { ctrl := gomock.NewController(t) sourceManager := sourceproviders_test.NewMockSourceManager(ctrl) From ea1177c6e32907ee86dbd8ebb07432386b5507b8 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 3 Nov 2026 18:55:50 +0000 Subject: [PATCH 04/70] feat(component): carry over rendered autorelease state Reuse the committed release base when an upstream autorelease spec has prior output marked with an RPMAUTOSPEC autorelease header. Expand the committed spec in mock and use the leading integer of Release as the -b parameter. Copy the committed changelog body into the working upstream changelog file before the existing version-change reset and remaining render processing. Use a temporary committed snapshot so uncommitted edits cannot affect the spec or its supporting macro files. Install rpm-build in the render chroot for rpmspec, update user and agent guidance, and add regressions for carryover, marker matching, version resets, query failures, and temporary snapshot cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3497f2b2-c7fb-456e-a58f-6889b6ec67e2 --- docs/user/how-to/preview-without-lockfile.md | 9 + .../app/azldev/agentskill/agentskill_test.go | 2 + .../content/refresh-upstream-commit.md.tmpl | 7 +- .../app/azldev/cmds/component/mockproc.go | 5 +- .../azldev/cmds/component/render_lockless.go | 80 ++++++++- .../render_lockless_internal_test.go | 160 +++++++++++++++++- .../app/azldev/core/sources/mockprocessor.go | 31 ++++ .../azldev/core/sources/mockprocessor_test.go | 51 ++++++ 8 files changed, 336 insertions(+), 9 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 0dde5b1db..93e574fc4 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -126,6 +126,15 @@ For an initial `%autorelease` render, it generates the complete upstream release, and adds or replaces the macro's `-b` parameter. These rpmautospec operations run inside the shared mock chroot. +For an existing output whose spec has a line starting with `## RPMAUTOSPEC:` +and containing `autorelease` on that same line, a `%autorelease` upstream spec +inherits its base from the committed rendered spec. The renderer expands the +committed `Release:` with `rpmspec` in mock, using a temporary copy of the +committed dist-git files, and uses its leading integer as `-b` (for example, +`9.fc43` becomes `-b 9`). It also replaces the working checkout's `changelog` +file with the committed `%changelog` body, excluding the section header. +The later version-change release reset still applies. + The last commit touching the component output is the **last render commit**. The renderer fully loads TOML, including includes and component defaults, at that commit and at the current project commit to compare `spec.upstream-commit`. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 2c5aedbbc..614615599 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -182,6 +182,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { require.NoError(t, err) assert.Contains(t, doc, "fully parsed committed TOML") assert.Contains(t, doc, "automatically commits only that directory") + assert.Contains(t, doc, "## RPMAUTOSPEC:") + assert.Contains(t, doc, "leading integer of the committed,") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 51fdf7f38..45ac4bbb5 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -59,7 +59,12 @@ Rendering preserves the committed downstream Release and changelog for static specs, resets Release to its original upstream value on a version change, and runs host `rpmdev-bumpspec -c` with `updated ` plus upstream changes. An initial `%autorelease` render generates the complete upstream -changelog and calculates its base release inside mock. Existing overlays, +changelog and calculates its base release inside mock. For existing output +marked by a line starting with `## RPMAUTOSPEC:` and containing `autorelease`, +an `%autorelease` upstream spec inherits the leading integer of the committed, +macro-expanded Release as `-b` and its committed `%changelog` body as the +working `changelog` file. Expansion uses the committed files inside mock; +the later version-change release reset still applies. Existing overlays, synthetic history, RPM expansion, and filtering still run afterward. Rendering then replaces `{{ .RenderedSpecsDir }}///` and diff --git a/internal/app/azldev/cmds/component/mockproc.go b/internal/app/azldev/cmds/component/mockproc.go index b11d57ce4..137c90f10 100644 --- a/internal/app/azldev/cmds/component/mockproc.go +++ b/internal/app/azldev/cmds/component/mockproc.go @@ -5,7 +5,8 @@ package component // Required-package presets for the shared MockProcessor. // -// Render needs rpmautospec (macro expansion), rpmdevtools (spectool), and git +// Render needs rpmautospec (macro expansion), rpmdevtools (spectool), rpm-build +// (rpmspec queries of committed Release values), and git // (required for rpmautospec to read commit history). python3-click is required // by rpmautospec but not declared as an RPM dependency. Ecosystem macro // packages (go-srpm-macros, etc.) are already present via @buildsys-build → @@ -15,7 +16,7 @@ package component // pulled in via @buildsys-build, but we install it explicitly so we don't // depend on a particular buildgroup composition. func mockPackagesForRender() []string { - return []string{"rpmautospec", "rpmdevtools", "git", "python3-click"} + return []string{"rpmautospec", "rpmdevtools", "rpm-build", "git", "python3-click"} } func mockPackagesForQuery() []string { diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index dc47c460f..1346ccf1e 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -38,6 +38,7 @@ type locklessRenderUpdate struct { type lastRender struct { spec string + tree *object.Tree previous string current string } @@ -45,6 +46,7 @@ type lastRender struct { type renderAutorelease interface { GenerateChangelog(ctx context.Context, dir, filename string) (string, error) CalculateBaseRelease(ctx context.Context, dir, filename string) (string, error) + ExpandRelease(ctx context.Context, dir, filename string) (string, error) } func prepareLocklessRender( @@ -97,7 +99,7 @@ func readLastRender( return history, err } - _, err = tree.Tree(path) + history.tree, err = tree.Tree(path) if errors.Is(err, object.ErrDirectoryNotFound) { return history, nil } @@ -292,10 +294,14 @@ func collectRenderUpstreamChanges( func locklessRenderRelease( env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, ) (string, error) { - if history.spec != "" || !sources.ReleaseUsesAutorelease(release) || env.DryRun() { + if !sources.ReleaseUsesAutorelease(release) || env.DryRun() { return release, nil } + if history.spec != "" { + return carryRenderedAutorelease(env, autospec, dir, filename, release, history) + } + changelog, err := autospec.GenerateChangelog(env, dir, filename) if err != nil { return "", fmt.Errorf("generating initial upstream changelog:\n%w", err) @@ -314,6 +320,72 @@ func locklessRenderRelease( return withAutoreleaseBase(release, calculated) } +var ( + renderedAutoreleaseMarker = regexp.MustCompile(`(?m)^## RPMAUTOSPEC:[^\r\n]*autorelease`) + renderedReleaseNumber = regexp.MustCompile(`^[0-9]+`) +) + +func carryRenderedAutorelease( + env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, +) (string, error) { + if !renderedAutoreleaseMarker.MatchString(history.spec) { + return release, nil + } + + expanded, err := expandCommittedRelease(env, autospec, dir, filename, history) + if err != nil { + return "", fmt.Errorf("expanding committed Release:\n%w", err) + } + + release, err = withAutoreleaseBase(release, renderedReleaseNumber.FindString(strings.TrimSpace(expanded))) + if err != nil { + return "", fmt.Errorf("using committed Release %#q as autorelease base:\n%w", expanded, err) + } + + previous := []byte(history.spec) + + start, end, err := renderChangelogBounds(previous, spec.EditorLegacy) + if err != nil { + return "", err + } + + _, changelog, _ := strings.Cut(string(previous[start:end]), "\n") + if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "changelog"), []byte(changelog), + fileperms.PublicFile); err != nil { + return "", fmt.Errorf("writing committed 'changelog' content:\n%w", err) + } + + return release, nil +} + +func expandCommittedRelease( + env *azldev.Env, autospec renderAutorelease, dir, filename string, history lastRender, +) (release string, err error) { + snapshot, err := fileutils.MkdirTemp(env.FS(), filepath.Dir(dir), "azldev-render-previous-") + if err != nil { + return "", fmt.Errorf("creating committed dist-git snapshot:\n%w", err) + } + defer fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), snapshot, &err) + + if history.tree != nil { + if err := copyTreeToFS(history.tree, env.FS(), snapshot); err != nil { + return "", fmt.Errorf("copying committed dist-git snapshot:\n%w", err) + } + } + + if err := fileutils.WriteFile(env.FS(), filepath.Join(snapshot, filename), []byte(history.spec), + fileperms.PublicFile); err != nil { + return "", fmt.Errorf("writing committed spec snapshot:\n%w", err) + } + + release, err = autospec.ExpandRelease(env, snapshot, filename) + if err != nil { + return "", fmt.Errorf("querying committed spec:\n%w", err) + } + + return release, nil +} + func restoreLocklessRelease( original, release, version string, history lastRender, upstreamChanged bool, ) (string, error) { @@ -358,14 +430,14 @@ var autoreleaseBaseArg = regexp.MustCompile(`(^|\s)-b(?:\s+|=)?[0-9]+`) func withAutoreleaseBase(release, output string) (string, error) { fields := strings.Fields(output) if len(fields) == 0 { - return "", errors.New("rpmautospec returned an empty release number") + return "", errors.New("empty autorelease base") } number := fields[len(fields)-1] value, err := strconv.Atoi(number) if err != nil || value < 1 { - return "", fmt.Errorf("rpmautospec returned invalid release number %#q", output) + return "", fmt.Errorf("invalid autorelease base %#q", output) } if autoreleaseBaseArg.MatchString(release) { diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 6279ebc27..1025b6b73 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "os/exec" + "path/filepath" "strconv" "strings" "testing" @@ -65,6 +66,12 @@ release-ver = "1" history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl") require.NoError(t, err) assert.Equal(t, oldSpec, history.spec) + require.NotNil(t, history.tree) + committed, err := history.tree.File("curl.spec") + require.NoError(t, err) + content, err := committed.Contents() + require.NoError(t, err) + assert.Equal(t, oldSpec, content) assert.Equal(t, "aaaaaaa", history.previous) assert.Equal(t, "bbbbbbb", history.current) } @@ -125,8 +132,9 @@ upstream-commit = "bbbbbbb" } type renderAutospecStub struct { - calls []string - err error + calls []string + err error + expand func(string, string) (string, error) } func (s *renderAutospecStub) GenerateChangelog(_ context.Context, _, _ string) (string, error) { @@ -141,6 +149,12 @@ func (s *renderAutospecStub) CalculateBaseRelease(_ context.Context, _, _ string return "Calculated release number: 42\n", s.err } +func (s *renderAutospecStub) ExpandRelease(_ context.Context, dir, filename string) (string, error) { + s.calls = append(s.calls, "expand") + + return s.expand(dir, filename) +} + func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { const oldChangelog = "* Previous downstream entry\n- Preserve spaces \n" for _, testCase := range []struct { @@ -255,6 +269,148 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { } } +const carriedAutoreleaseChangelog = "* Mon Nov 02 2026 Packager - 1-9\n- Preserve spaces \n\n" + +func testCommittedAutoreleaseHistory(t *testing.T) lastRender { + t.Helper() + + content := "## RPMAUTOSPEC: processed with autorelease, autochangelog\n%global carried_release 9\n" + + renderTestSpec("1", "%{carried_release}%{?dist}", carriedAutoreleaseChangelog) + + "%files\n/file\n" + repo, commits := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + "curl.spec": []byte(content), "macros.inc": []byte("committed macro definitions"), + }}}) + tree, err := resolveTree(repo, commits[0]) + require.NoError(t, err) + + return lastRender{spec: content, tree: tree, previous: "old", current: "old"} +} + +func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { + for _, testCase := range []struct { + name string + version string + upstreamChanged bool + wantRelease string + }{ + {name: "same upstream", version: "1", wantRelease: "%autorelease -b 9 -e test"}, + { + name: "new upstream same version", version: "1", upstreamChanged: true, + wantRelease: "%autorelease -b 9 -e test", + }, + { + name: "version reset", version: "2", upstreamChanged: true, + wantRelease: "%autorelease -b 2 -e test", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + history := testCommittedAutoreleaseHistory(t) + if testCase.upstreamChanged { + history.current = "new" + } + + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(renderTestSpec(testCase.version, "%autorelease -b 2 -e test", "%autochangelog\n")), + fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", []byte("stale"), + fileperms.PublicFile)) + + var snapshot string + + stub := &renderAutospecStub{expand: func(dir, filename string) (string, error) { + snapshot = dir + content, err := fileutils.ReadFile(env.TestFS, filepath.Join(dir, filename)) + require.NoError(t, err) + assert.Equal(t, history.spec, string(content)) + content, err = fileutils.ReadFile(env.TestFS, filepath.Join(dir, "macros.inc")) + require.NoError(t, err) + assert.Equal(t, "committed macro definitions", string(content)) + + return " 9.fc43\n", nil + }} + + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return "abc123 Upstream fix\n", nil + } + bumped := false + env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + bumped = true + content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: "+testCase.wantRelease+"\n") + assert.Contains(t, string(content), "%changelog\n%autochangelog\n") + content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, carriedAutoreleaseChangelog, string(content)) + + return nil + } + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, + history, &locklessRenderUpdate{subject: "updated curl"}) + require.NoError(t, err) + assert.True(t, bumped) + assert.Equal(t, []string{"expand"}, stub.calls) + assert.NotEqual(t, "/working", snapshot) + exists, err := fileutils.DirExists(env.TestFS, snapshot) + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + +func TestLocklessRenderRelease_RequiresAutoreleaseMarker(t *testing.T) { + for _, testCase := range []struct{ header, release string }{ + {"## RPMAUTOSPEC: autochangelog\n", "%autorelease"}, + {"## RPMAUTOSPEC: autochangelog\nautorelease\n", "%autorelease"}, + {" ## RPMAUTOSPEC: autorelease\n", "%autorelease"}, + {"## RPMAUTOSPEC: autorelease\n", "1%{?dist}"}, + } { + env := testutils.NewTestEnv(t) + stub := &renderAutospecStub{} + history := lastRender{spec: testCase.header + renderTestSpec("1", "9", "old\n")} + got, err := locklessRenderRelease(env.Env, stub, "/working", "curl.spec", testCase.release, history) + require.NoError(t, err) + assert.Equal(t, testCase.release, got) + assert.Empty(t, stub.calls) + } +} + +func TestPrepareLocklessSpec_CommittedAutoreleaseFailure(t *testing.T) { + for _, testCase := range []struct { + name, output, want string + err error + }{ + {name: "query failure", err: errors.New("query failed"), want: "expanding committed Release"}, + {name: "unresolved release", output: "%{unknown}", want: "using committed Release"}, + {name: "zero base", output: "0.fc43", want: "using committed Release"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + original := renderTestSpec("1", "%autorelease", "%autochangelog\n") + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(original), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", + []byte("original changelog"), fileperms.PublicFile)) + + stub := &renderAutospecStub{expand: func(_, _ string) (string, error) { + return testCase.output, testCase.err + }} + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, + testCommittedAutoreleaseHistory(t), &locklessRenderUpdate{subject: "updated curl"}) + require.ErrorContains(t, err, testCase.want) + content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, err) + assert.Equal(t, original, string(content)) + content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, "original changelog", string(content)) + }) + } +} + func TestWithAutoreleaseBase(t *testing.T) { for _, testCase := range []struct{ release, want string }{ {"%autorelease", "%autorelease -b 42"}, diff --git a/internal/app/azldev/core/sources/mockprocessor.go b/internal/app/azldev/core/sources/mockprocessor.go index a1d8410a8..270244962 100644 --- a/internal/app/azldev/core/sources/mockprocessor.go +++ b/internal/app/azldev/core/sources/mockprocessor.go @@ -375,6 +375,37 @@ func (p *MockProcessor) GenerateChangelog(ctx context.Context, specHostDir, spec return p.runRPMAutospec(ctx, specHostDir, specFilename, "generate-changelog") } +// ExpandRelease queries the spec's macro-expanded Release inside the shared mock chroot. +func (p *MockProcessor) ExpandRelease(ctx context.Context, specHostDir, specFilename string) (string, error) { + p.mu.Lock() + defer p.mu.Unlock() + + if err := p.initOnce(ctx); err != nil { + return "", err + } + + runner := p.runner.Clone() + runner.WithUnprivileged() + runner.AddBindMount(specHostDir, chrootProvenanceDir) + + cmd, err := runner.CmdInChroot(ctx, []string{ + "rpmspec", "-q", "--srpm", "--queryformat", "%{RELEASE}", + "--define", "_sourcedir " + chrootProvenanceDir, + "--define", "_specdir " + chrootProvenanceDir, + filepath.Join(chrootProvenanceDir, specFilename), + }, false) + if err != nil { + return "", fmt.Errorf("creating committed Release query in mock chroot:\n%w", err) + } + + out, err := cmd.RunAndGetOutput(ctx) + if err != nil { + return "", fmt.Errorf("querying committed Release in mock chroot:\n%w", err) + } + + return out, nil +} + func (p *MockProcessor) runRPMAutospec( ctx context.Context, specHostDir, specFilename string, args ...string, ) (string, error) { diff --git a/internal/app/azldev/core/sources/mockprocessor_test.go b/internal/app/azldev/core/sources/mockprocessor_test.go index a38debcf6..3bb2da3a7 100644 --- a/internal/app/azldev/core/sources/mockprocessor_test.go +++ b/internal/app/azldev/core/sources/mockprocessor_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + "github.com/kballard/go-shellquote" "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" "github.com/stretchr/testify/assert" @@ -260,6 +261,56 @@ func TestRenderRPMAutospecCommands(t *testing.T) { } } +func TestExpandRelease(t *testing.T) { + const initFailure = "initialization failure" + for _, phase := range []string{"success", initFailure, "query failure"} { + t.Run(phase, func(t *testing.T) { + ctx := newMockProcessorTestCtx() + ctx.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + if phase == initFailure { + return errInitBoom + } + + return nil + } + queried := false + ctx.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + queried = true + joined := strings.Join(cmd.Args, " ") + assert.Contains(t, joined, "--unpriv") + assert.Contains(t, joined, `bind_mount:dirs=[("/committed", "/tmp/provenance")]`) + assert.Contains(t, joined, "rpmspec -q --srpm --queryformat") + + args, err := shellquote.Split(cmd.Args[len(cmd.Args)-1]) + require.NoError(t, err) + assert.Contains(t, args, "%{RELEASE}") + assert.Contains(t, joined, "_sourcedir /tmp/provenance") + assert.Contains(t, joined, "/tmp/provenance/curl.spec") + + if phase == "query failure" { + return "", errRpmautospecExit + } + + return "9.fc43\n", nil + } + processor := NewMockProcessor(ctx, "/mock/config") + out, err := processor.ExpandRelease(ctx, "/committed", "curl.spec") + + assert.Equal(t, phase != initFailure, queried) + + switch phase { + case initFailure: + require.ErrorIs(t, err, errInitBoom) + case "query failure": + require.ErrorIs(t, err, errRpmautospecExit) + default: + require.NoError(t, err) + assert.Equal(t, "9.fc43\n", out) + } + }) + } +} + func TestCalculateRelease_InitFailurePropagates(t *testing.T) { t.Parallel() From 7c225c8c8423d23cacb00dc322642776d81fc208 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 3 Nov 2026 22:20:46 +0000 Subject: [PATCH 05/70] fix(component): ignore RPM diagnostics when querying render release Prefix the rpmspec release query and select only its labelled result. Mock can merge non-fatal RPM diagnostics into stdout, so treating the whole output as a release prevented the emacs lockless render from carrying over its existing autorelease base. Cover leading and trailing diagnostics and missing query values. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3497f2b2-c7fb-456e-a58f-6889b6ec67e2 --- .../app/azldev/core/sources/mockprocessor.go | 10 ++++++++-- .../azldev/core/sources/mockprocessor_test.go | 20 +++++++++++++------ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/internal/app/azldev/core/sources/mockprocessor.go b/internal/app/azldev/core/sources/mockprocessor.go index 270244962..ddfbce97e 100644 --- a/internal/app/azldev/core/sources/mockprocessor.go +++ b/internal/app/azldev/core/sources/mockprocessor.go @@ -389,7 +389,7 @@ func (p *MockProcessor) ExpandRelease(ctx context.Context, specHostDir, specFile runner.AddBindMount(specHostDir, chrootProvenanceDir) cmd, err := runner.CmdInChroot(ctx, []string{ - "rpmspec", "-q", "--srpm", "--queryformat", "%{RELEASE}", + "rpmspec", "-q", "--srpm", "--queryformat", "release=%{RELEASE}\n", "--define", "_sourcedir " + chrootProvenanceDir, "--define", "_specdir " + chrootProvenanceDir, filepath.Join(chrootProvenanceDir, specFilename), @@ -403,7 +403,13 @@ func (p *MockProcessor) ExpandRelease(ctx context.Context, specHostDir, specFile return "", fmt.Errorf("querying committed Release in mock chroot:\n%w", err) } - return out, nil + for _, line := range strings.Split(out, "\n") { + if release, found := strings.CutPrefix(strings.TrimSpace(line), "release="); found && release != "" { + return release, nil + } + } + + return "", fmt.Errorf("no Release field in committed spec query output %#q", out) } func (p *MockProcessor) runRPMAutospec( diff --git a/internal/app/azldev/core/sources/mockprocessor_test.go b/internal/app/azldev/core/sources/mockprocessor_test.go index 3bb2da3a7..099a99cd6 100644 --- a/internal/app/azldev/core/sources/mockprocessor_test.go +++ b/internal/app/azldev/core/sources/mockprocessor_test.go @@ -263,7 +263,7 @@ func TestRenderRPMAutospecCommands(t *testing.T) { func TestExpandRelease(t *testing.T) { const initFailure = "initialization failure" - for _, phase := range []string{"success", initFailure, "query failure"} { + for _, phase := range []string{"success", "diagnostics", "missing release", initFailure, "query failure"} { t.Run(phase, func(t *testing.T) { ctx := newMockProcessorTestCtx() ctx.CmdFactory.RunHandler = func(_ *exec.Cmd) error { @@ -283,15 +283,21 @@ func TestExpandRelease(t *testing.T) { args, err := shellquote.Split(cmd.Args[len(cmd.Args)-1]) require.NoError(t, err) - assert.Contains(t, args, "%{RELEASE}") + assert.Contains(t, args, "release=%{RELEASE}\n") assert.Contains(t, joined, "_sourcedir /tmp/provenance") assert.Contains(t, joined, "/tmp/provenance/curl.spec") - if phase == "query failure" { + switch phase { + case "query failure": return "", errRpmautospecExit + case "diagnostics": + return "error: %changelog not in descending chronological order\n" + + "release=9.fc43\nwarning: trailing diagnostic\n", nil + case "missing release": + return "error: %changelog not in descending chronological order\nrelease=\n", nil + default: + return "release=9.fc43\n", nil } - - return "9.fc43\n", nil } processor := NewMockProcessor(ctx, "/mock/config") out, err := processor.ExpandRelease(ctx, "/committed", "curl.spec") @@ -303,9 +309,11 @@ func TestExpandRelease(t *testing.T) { require.ErrorIs(t, err, errInitBoom) case "query failure": require.ErrorIs(t, err, errRpmautospecExit) + case "missing release": + require.ErrorContains(t, err, "no Release field") default: require.NoError(t, err) - assert.Equal(t, "9.fc43\n", out) + assert.Equal(t, "9.fc43", out) } }) } From 442de33d3030f22d58b3c0d43b48ef8b196653b7 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Tue, 3 Nov 2026 22:40:08 +0000 Subject: [PATCH 06/70] fix(component): keep lockless rendering on the host Separate lockless rendering from the locked mock processing pipeline. Run release, changelog, provenance, and sidecar queries on the host, without requiring or initializing mock for an all-lockless selection. Mixed renders send only locked components through mock. Do not run rpmautospec process-distgit for lockless components. Preserve the raw spec macros and changelog sidecar while retaining the existing release carryover, overlays, filtering, and output commit lifecycle. Add regressions for host tool queries, diagnostic handling, macro and changelog preservation, and mixed-mode routing and failures. Update user and emitted agent guidance to describe the host-only workflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3497f2b2-c7fb-456e-a58f-6889b6ec67e2 --- docs/user/how-to/preview-without-lockfile.md | 20 +- .../reference/cli/azldev_component_render.md | 6 +- docs/user/reference/config/components.md | 2 +- .../app/azldev/agentskill/agentskill_test.go | 2 + .../azldev/agentskill/content/azldev.md.tmpl | 7 +- .../content/refresh-upstream-commit.md.tmpl | 12 +- .../content/withoutlockfile/azldev.md.tmpl | 6 +- .../content/withoutlockfile/comp-toml.md.tmpl | 8 +- .../app/azldev/cmds/component/mockproc.go | 5 +- .../azldev/cmds/component/mockprocessor.go | 20 ++ internal/app/azldev/cmds/component/render.go | 44 ++-- .../app/azldev/cmds/component/render_host.go | 117 ++++++++++ .../component/render_host_internal_test.go | 216 ++++++++++++++++++ .../component/render_modes_internal_test.go | 83 ++++++- .../app/azldev/core/sources/mockprocessor.go | 57 +---- .../azldev/core/sources/mockprocessor_test.go | 88 ------- .../app/azldev/core/sources/sourceprep.go | 19 +- .../core/sources/upstream_provenance.go | 15 +- 18 files changed, 531 insertions(+), 196 deletions(-) create mode 100644 internal/app/azldev/cmds/component/render_host.go create mode 100644 internal/app/azldev/cmds/component/render_host_internal_test.go diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 93e574fc4..b3c0871d2 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -111,8 +111,9 @@ produce a synthetic commit. ## Render and Commit an Upstream Component The following process applies only to **upstream components selected as -lockless**. Local components and components using lock files retain their -existing rendering behavior. +lockless**. Local components skip the upstream release/changelog carryover and +automatic commit steps. All lockless rendering runs on the host without mock; +components using lock files retain their existing mock-based rendering. The renderer clones the selected upstream dist-git to temporary storage. It finds the existing committed output under @@ -124,12 +125,12 @@ carries over the committed rendered spec's exact `Release:` value and For an initial `%autorelease` render, it generates the complete upstream `changelog` file (replacing any existing file), calculates the numeric upstream release, and adds or replaces the macro's `-b` parameter. These rpmautospec -operations run inside the shared mock chroot. +operations run on the host; they do not preprocess or rewrite the spec. For an existing output whose spec has a line starting with `## RPMAUTOSPEC:` and containing `autorelease` on that same line, a `%autorelease` upstream spec inherits its base from the committed rendered spec. The renderer expands the -committed `Release:` with `rpmspec` in mock, using a temporary copy of the +committed `Release:` with host `rpmspec`, using a temporary copy of the committed dist-git files, and uses its leading integer as `-b` (for example, `9.fc43` becomes `-b 9`). It also replaces the working checkout's `changelog` file with the committed `%changelog` body, excluding the section header. @@ -149,14 +150,19 @@ version (or there is no previous output), it restores the original upstream `updated ` followed by upstream changes on separate lines. The tool's no-op for `%autorelease` specs is accepted. -Existing overlays, synthetic history, RPM expansion, and sidecar filtering run -after these steps. The renderer replaces the component output, excluding `.git`, +Existing overlays, synthetic history, and sidecar filtering run after these +steps. Host `spectool` supplies the sidecar keep-list, with `changelog` always +preserved. The spec is **not** preprocessed with `rpmautospec process-distgit`: +`%autorelease`, `%autochangelog`, and other RPM macros remain unexpanded. +No mock configuration is needed for lockless rendering. +The renderer replaces the component output, excluding `.git`, and commits only that directory. The subject is `updated `; the body lists upstream changes, if any. Unrelated staged files are not committed. Unchanged Git content does not produce an empty commit. A commit failure leaves the rendered files available for inspection. -Install `rpmdevtools` on the host and configure Git author identity before +Install `rpmdevtools`, `rpm-build`, and `rpmautospec` on the host as needed for +the release/changelog queries, and configure Git author identity before rendering. Output (including a custom `--output-dir`) must be inside the project Git repository. `--check-only` prepares and compares temporary output but never replaces rendered files or creates project commits. Because ordinary release diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 2c3175753..5bcdb88ca 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -34,7 +34,11 @@ the project repository, without including unrelated staged files. The output must be inside that repository; Git author identity must be configured and rpmdev-bumpspec must be installed. Commit updated upstream-commit TOML before rendering. --check-only never replaces output or creates project commits. -Local components and components using lock files keep their existing behavior. +Lockless rendering runs entirely on the host: it never uses mock or +rpmautospec process-distgit. The rendered spec retains its RPM macros, including +%autorelease and %autochangelog. Host spectool supplies the sidecar keep-list; +rpmautospec generate-changelog/calculate-release are used only when needed for +release history or provenance. Locked components keep their existing behavior. When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index cb925c80d..3fa997c9b 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -242,7 +242,7 @@ emit-upstream-provenance = true The values are read from the upstream spec **before** any azldev overlays are applied, so they reflect the true upstream Name-Version-Release, not the Azure Linux–modified spec. The macros are derived fresh at render/build time from the pinned upstream commit and emitted into the component's generated macros file (loaded via `%{load:...}`). -For specs whose `Release` uses rpmautospec (`Release: %autorelease`), the pristine Fedora release number is computed by running `rpmautospec calculate-release` in the project distro's mock chroot against the upstream dist-git checkout (whose history is Fedora's, not azldev's synthetic overlay history). This keeps `rpmautospec` out of azldev's host dependencies. If the project distro has no mock config, or mock resolution fails, `%fedora_upstream_release` is skipped (with a warning) rather than emitting the literal `%autorelease`. +For specs whose `Release` uses rpmautospec (`Release: %autorelease`), the pristine Fedora release number is computed by running `rpmautospec calculate-release` against the upstream dist-git checkout (whose history is Fedora's, not azldev's synthetic overlay history). Lockless rendering runs this query on the host; other workflows use the project distro's mock chroot. If resolution is unavailable or fails, `%fedora_upstream_release` is skipped (with a warning) rather than emitting the literal `%autorelease`. Example: for a component pinned to Fedora 43's `grub2-2.12-5.fc43`, the spec can reference: diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 614615599..7941b6934 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -176,6 +176,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { require.NoError(t, err) assert.Contains(t, doc, "host `rpmdev-bumpspec`") assert.Contains(t, doc, "`--check-only` never creates project commits") + assert.Contains(t, doc, "never uses mock or `rpmautospec process-distgit`") } doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) @@ -184,6 +185,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "automatically commits only that directory") assert.Contains(t, doc, "## RPMAUTOSPEC:") assert.Contains(t, doc, "leading integer of the committed,") + assert.Contains(t, doc, "retains `%autorelease`, `%autochangelog`") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 136316f01..747269ca9 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -40,11 +40,14 @@ generated upstream-commit TOML instead. All others use locks. For listed upstream components, rendering carries release/changelog state from the committed output, runs host `rpmdev-bumpspec`, then applies the existing -overlays, synthetic history, RPM expansion, and filtering. It replaces and +overlays, synthetic history, and host `spectool` filtering. Lockless rendering +never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. +It preserves the `changelog` sidecar and replaces and automatically commits only that component's output directory; unrelated staged files are excluded. Commit refreshed TOML before rendering. Configure Git author identity and install `rpmdevtools`. `--check-only` never creates project commits. -Local and locked components retain their existing render behavior. +Local components skip upstream carryover and automatic commits; locked +components retain their existing render behavior. Top-level commands in this build: diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 45ac4bbb5..458573c74 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -59,19 +59,23 @@ Rendering preserves the committed downstream Release and changelog for static specs, resets Release to its original upstream value on a version change, and runs host `rpmdev-bumpspec -c` with `updated ` plus upstream changes. An initial `%autorelease` render generates the complete upstream -changelog and calculates its base release inside mock. For existing output +changelog and calculates its base release on the host. For existing output marked by a line starting with `## RPMAUTOSPEC:` and containing `autorelease`, an `%autorelease` upstream spec inherits the leading integer of the committed, macro-expanded Release as `-b` and its committed `%changelog` body as the -working `changelog` file. Expansion uses the committed files inside mock; +working `changelog` file. Expansion uses the committed files on the host; the later version-change release reset still applies. Existing overlays, -synthetic history, RPM expansion, and filtering still run afterward. +synthetic history, and host `spectool` filtering still run afterward. +Lockless rendering never uses mock or `rpmautospec process-distgit`; the output +retains `%autorelease`, `%autochangelog`, and other RPM macros unexpanded, and +preserves the `changelog` sidecar. Rendering then replaces `{{ .RenderedSpecsDir }}///` and automatically commits only that directory. The subject is `updated ` and the body contains upstream changes, if any. It never includes unrelated staged files or creates empty commits. Configure -Git author identity and install host `rpmdevtools`. `--check-only` never replaces +Git author identity and install host `rpmdevtools`, `rpm-build`, and `rpmautospec` +as needed. `--check-only` never replaces output or commits, but a new static release bump may cause drift without new upstream changes. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 6bf8cf159..ab4604a86 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -37,7 +37,11 @@ For upstream components, rendering preserves committed downstream release and changelog state before running host `rpmdev-bumpspec` and existing processing. It replaces and automatically commits only the component output directory. Commit refreshed TOML before rendering; configure Git author identity and install -`rpmdevtools`. `--check-only` never creates project commits. Local rendering is unchanged. +`rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. +Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros +remain unexpanded and the `changelog` sidecar is preserved. +`--check-only` never creates project commits. Local components skip upstream +carryover and automatic commits. Top-level commands in this build: diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 6ddb3295e..d60727f07 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -88,8 +88,12 @@ calculation = "manual" Upstream rendering always runs host `rpmdev-bumpspec -c` before the existing processing and automatically commits the component output directory. Commit -input TOML first, install host `rpmdevtools`, and configure Git author identity. -`--check-only` never creates project commits. Local rendering is unchanged. +input TOML first, install host `rpmdevtools`, `rpm-build`, and `rpmautospec` as +needed, and configure Git author identity. Lockless rendering never uses mock +or `rpmautospec process-distgit`; RPM macros remain unexpanded and the +`changelog` sidecar is preserved. +`--check-only` never creates project commits. Local components skip upstream +carryover and automatic commits. azldev reads `lockless-components.txt` beside the top-level `azldev.toml`. Listed component names use the lockless workflow for every command; diff --git a/internal/app/azldev/cmds/component/mockproc.go b/internal/app/azldev/cmds/component/mockproc.go index 137c90f10..929c07476 100644 --- a/internal/app/azldev/cmds/component/mockproc.go +++ b/internal/app/azldev/cmds/component/mockproc.go @@ -5,8 +5,7 @@ package component // Required-package presets for the shared MockProcessor. // -// Render needs rpmautospec (macro expansion), rpmdevtools (spectool), rpm-build -// (rpmspec queries of committed Release values), and git +// Locked render needs rpmautospec (macro expansion), rpmdevtools (spectool), and git // (required for rpmautospec to read commit history). python3-click is required // by rpmautospec but not declared as an RPM dependency. Ecosystem macro // packages (go-srpm-macros, etc.) are already present via @buildsys-build → @@ -16,7 +15,7 @@ package component // pulled in via @buildsys-build, but we install it explicitly so we don't // depend on a particular buildgroup composition. func mockPackagesForRender() []string { - return []string{"rpmautospec", "rpmdevtools", "rpm-build", "git", "python3-click"} + return []string{"rpmautospec", "rpmdevtools", "git", "python3-click"} } func mockPackagesForQuery() []string { diff --git a/internal/app/azldev/cmds/component/mockprocessor.go b/internal/app/azldev/cmds/component/mockprocessor.go index 7904c13fa..59de5e927 100644 --- a/internal/app/azldev/cmds/component/mockprocessor.go +++ b/internal/app/azldev/cmds/component/mockprocessor.go @@ -5,11 +5,13 @@ package component import ( "context" + "errors" "log/slog" "path/filepath" "time" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" ) @@ -22,6 +24,24 @@ const buildProvenanceMockSubdir = "azldev-provenance-mock" // scrubbing the mock chroot, so a hung scrub can't block command shutdown. const mockProcessorCleanupTimeout = 5 * time.Minute +func renderMockProcessor(env *azldev.Env, comps []components.Component) (*sources.MockProcessor, error) { + for _, comp := range comps { + if env.ComponentWithoutLockfile(comp.GetName()) { + continue + } + + processor := createMockProcessor(env) + if processor == nil { + return nil, errors.New( + "mock config required for locked rendering; ensure the project has a valid distro with mock config") + } + + return processor, nil + } + + return nil, nil //nolint:nilnil // All-lockless renders do not need a mock processor. +} + // createMockProcessor creates a lazily initialized [sources.MockProcessor] // using the project/build distro's mock config. Source providers may use a // different per-component upstream distro, which must not select the build diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index f8e9637cd..3820cd9f4 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -83,7 +83,11 @@ the project repository, without including unrelated staged files. The output must be inside that repository; Git author identity must be configured and rpmdev-bumpspec must be installed. Commit updated upstream-commit TOML before rendering. --check-only never replaces output or creates project commits. -Local components and components using lock files keep their existing behavior. +Lockless rendering runs entirely on the host: it never uses mock or +rpmautospec process-distgit. The rendered spec retains its RPM macros, including +%autorelease and %autochangelog. Host spectool supplies the sidecar keep-list; +rpmautospec generate-changelog/calculate-release are used only when needed for +release history or provenance. Locked components keep their existing behavior. When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to @@ -172,7 +176,7 @@ const ( // RenderComponents renders the post-overlay spec and sidecar files for each // selected component into the output directory. Processing is done in four phases: // 1. Parallel source preparation (clone, overlay, synthetic git) -// 2. Batch mock processing (rpmautospec + spectool in a single chroot call) +// 2. Host sidecar queries for lockless components; batch mock processing for locked components // 3. Parallel finishing (filter files, remove .git, copy output) // 4. Serial project commits for lockless upstream component output func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, error) { @@ -197,11 +201,11 @@ func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, ) } - // Create mock processor for rpmautospec/spectool. - mockProcessor := createMockProcessor(env) - if mockProcessor == nil { - return nil, errors.New( - "mock config required for rendering; ensure the project has a valid distro with mock config") + componentList := comps.Components() + + mockProcessor, err := renderMockProcessor(env, componentList) + if err != nil { + return nil, err } defer destroyMockProcessor(env, mockProcessor) @@ -225,14 +229,13 @@ func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, } }() - componentList := comps.Components() results := make([]*RenderResult, len(componentList)) // ── Phase 1: Parallel source preparation ── prepared := parallelPrepare(env, mockProcessor, componentList, stagingDir, options, results) - // ── Phase 2: Batch mock processing ── - mockResultMap := batchMockProcess(env, mockProcessor, stagingDir, prepared) + // ── Phase 2: Mode-specific source processing ── + mockResultMap := processRenderSources(env, mockProcessor, stagingDir, prepared) // Prune orphan component dirs (components removed from config) when // --clean-stale is set. Per-component output dirs that match the resolved @@ -392,6 +395,7 @@ type preparedComponent struct { specFilename string // e.g., "curl.spec" compOutputDir string // validated output path computed in phase 1 update *locklessRenderUpdate + lockless bool } // prepResult pairs a prepared component (on success) or a render result (on error). @@ -565,8 +569,12 @@ func prepareComponentSources( preparerOpts := append(gitRepoPreparerOptionsForMode(env, distro, lockless), sources.WithSkipLookaside(), sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), - sources.WithMockProcessor(mockProcessor), ) + if lockless { + preparerOpts = append(preparerOpts, sources.WithAutoreleaseResolver(hostRenderProcessor{env: env})) + } else { + preparerOpts = append(preparerOpts, sources.WithMockProcessor(mockProcessor)) + } var update *locklessRenderUpdate @@ -575,7 +583,8 @@ func prepareComponentSources( func(_ context.Context, fetched components.Component, dir string) error { var prepErr error //nolint:contextcheck // The callback context is the captured preparation env. - update, prepErr = prepareLocklessRender(env, mockProcessor, fetched, dir, compOutputDir, specEditor) + update, prepErr = prepareLocklessRender(env, hostRenderProcessor{env: env}, + fetched, dir, compOutputDir, specEditor) return prepErr })) @@ -607,6 +616,7 @@ func prepareComponentSources( comp: comp, specFilename: filepath.Base(specPath), update: update, + lockless: lockless, }, nil } @@ -617,7 +627,7 @@ func prepareComponentSources( // batchMockProcess runs rpmautospec and spectool for all prepared components in // a single mock chroot invocation. Returns a map from component name to result. // -// NOTE: All components share one mock chroot initialized from the project's +// NOTE: All locked components share one mock chroot initialized from the project's // default distro. Phase 1 resolves distro per-component for source fetching, // but the mock environment (macros, rpmautospec version, etc.) is uniform. // This matches the current Koji build model where all components target the @@ -661,7 +671,7 @@ func batchMockProcess( // Phase 3: Parallel finishing // ────────────────────────────────────────────────────────────────────────────── -// parallelFinish applies mock results (file filtering, .git removal) and copies +// parallelFinish applies source-query results (file filtering, .git removal) and copies // rendered output for all successfully prepared components. In --check-only // mode, the copy step is replaced with a tree comparison and no disk writes // happen. @@ -781,7 +791,7 @@ func finishComponentRender( componentDir := filepath.Join(stagingDir, componentName) specPath := filepath.Join(componentDir, prep.specFilename) - // Check mock result. + // Check the host or mock source-query result. mockResult, hasMockResult := mockResultMap[componentName] if !hasMockResult { return false, fmt.Errorf( @@ -790,10 +800,10 @@ func finishComponentRender( if mockResult.Error != nil { return false, fmt.Errorf( - "mock processing failed for %#q:\n%w", componentName, mockResult.Error) + "source processing failed for %#q:\n%w", componentName, mockResult.Error) } - // Filter files using spectool result from batch mock. + // Filter files using the host or mock spectool result. // Skip filtering when: // 1. The component config explicitly opts out via 'skip-file-filter'. // 2. spectool output contains unexpanded RPM macros (%{...}), indicating diff --git a/internal/app/azldev/cmds/component/render_host.go b/internal/app/azldev/cmds/component/render_host.go new file mode 100644 index 000000000..4f47c32c8 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_host.go @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "path/filepath" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spectool" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" +) + +type hostRenderProcessor struct { + env *azldev.Env +} + +func (p hostRenderProcessor) GenerateChangelog(ctx context.Context, dir, filename string) (string, error) { + return p.run(ctx, dir, "rpmautospec", "generate-changelog", filename) +} + +func (p hostRenderProcessor) CalculateBaseRelease(ctx context.Context, dir, filename string) (string, error) { + return p.run(ctx, dir, "rpmautospec", "calculate-release", "--number-only", filename) +} + +func (p hostRenderProcessor) CalculateRelease(ctx context.Context, dir, filename string) (string, error) { + return p.run(ctx, dir, "rpmautospec", "calculate-release", "--complete-release", filename) +} + +func (p hostRenderProcessor) ExpandRelease(ctx context.Context, dir, filename string) (string, error) { + out, err := p.run(ctx, dir, "rpmspec", + "-q", "--srpm", "--queryformat", "release=%{RELEASE}\n", + "--define", "_sourcedir "+dir, "--define", "_specdir "+dir, filename) + if err != nil { + return "", err + } + + for _, line := range strings.Split(out, "\n") { + if release, found := strings.CutPrefix(strings.TrimSpace(line), "release="); found && release != "" { + return release, nil + } + } + + return "", fmt.Errorf("no Release field in committed spec query output %#q", out) +} + +func (p hostRenderProcessor) specFiles(ctx context.Context, dir, filename string) ([]string, error) { + out, err := p.run(ctx, dir, "spectool", "--define", "_sourcedir "+dir, "-l", "-a", filename) + if err != nil { + return nil, err + } + + return append(spectool.ParseSpectoolOutput(out), "changelog"), nil +} + +func (p hostRenderProcessor) run(ctx context.Context, dir, binary string, args ...string) (string, error) { + if err := prereqs.RequireExecutable(p.env, binary, nil); err != nil { + return "", fmt.Errorf("lockless rendering requires host program %#q:\n%w", binary, err) + } + + raw := exec.CommandContext(ctx, binary, args...) + raw.Dir = dir + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + cmd, err := p.env.Command(raw) + if err != nil { + return "", fmt.Errorf("creating host %#q command:\n%w", binary, err) + } + + out, err := cmd.RunAndGetOutput(ctx) + if err != nil { + return "", fmt.Errorf("running host %#q (%s):\n%w", binary, stderr.String(), err) + } + + return out, nil +} + +func processRenderSources( + env *azldev.Env, processor *sources.MockProcessor, stagingDir string, prepared []*preparedComponent, +) map[string]*sources.ComponentMockResult { + var locked []*preparedComponent + + results := make(map[string]*sources.ComponentMockResult, len(prepared)) + host := hostRenderProcessor{env: env} + + for _, prep := range prepared { + if !prep.lockless { + locked = append(locked, prep) + + continue + } + + name := prep.comp.GetName() + + result := &sources.ComponentMockResult{Name: name} + if !prep.comp.GetConfig().Render.SkipFileFilter { + result.SpecFiles, result.Error = host.specFiles(env, filepath.Join(stagingDir, name), prep.specFilename) + } + + results[name] = result + } + + for name, result := range batchMockProcess(env, processor, stagingDir, locked) { + results[name] = result + } + + return results +} diff --git a/internal/app/azldev/cmds/component/render_host_internal_test.go b/internal/app/azldev/cmds/component/render_host_internal_test.go new file mode 100644 index 000000000..819469312 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_host_internal_test.go @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "os/exec" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestHostRenderAutoreleaseCommands(t *testing.T) { + for _, operation := range []string{"changelog", "base", "provenance"} { + t.Run(operation, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RegisterCommandInSearchPath("rpmautospec") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, "/working", cmd.Dir) + + switch operation { + case "changelog": + assert.Equal(t, []string{"rpmautospec", "generate-changelog", "curl.spec"}, cmd.Args) + case "base": + assert.Equal(t, []string{"rpmautospec", "calculate-release", "--number-only", "curl.spec"}, cmd.Args) + case "provenance": + assert.Equal(t, []string{"rpmautospec", "calculate-release", "--complete-release", "curl.spec"}, cmd.Args) + } + + return "unchanged output\n", nil + } + processor := hostRenderProcessor{env: env.Env} + + var ( + out string + err error + ) + + switch operation { + case "changelog": + out, err = processor.GenerateChangelog(env.Env, "/working", "curl.spec") + case "base": + out, err = processor.CalculateBaseRelease(env.Env, "/working", "curl.spec") + case "provenance": + out, err = processor.CalculateRelease(env.Env, "/working", "curl.spec") + } + + require.NoError(t, err) + assert.Equal(t, "unchanged output\n", out) + }) + } +} + +func TestHostRenderExpandRelease(t *testing.T) { + const missingTool = "missing tool" + + for _, phase := range []string{"diagnostics", "missing release", "query failure", missingTool} { + t.Run(phase, func(t *testing.T) { + env := testutils.NewTestEnv(t) + if phase != missingTool { + env.CmdFactory.RegisterCommandInSearchPath("rpmspec") + } + + queryErr := errors.New("query failed") + called := false + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + called = true + + assert.Equal(t, "/committed", cmd.Dir) + assert.Equal(t, []string{ + "rpmspec", "-q", "--srpm", "--queryformat", "release=%{RELEASE}\n", + "--define", "_sourcedir /committed", "--define", "_specdir /committed", "curl.spec", + }, cmd.Args) + + switch phase { + case "query failure": + return "", queryErr + case "missing release": + return "error: %changelog not in descending chronological order\nrelease=\n", nil + default: + return "error: %changelog not in descending chronological order\n" + + "release=9.fc43\nwarning: trailing diagnostic\n", nil + } + } + out, err := (hostRenderProcessor{env: env.Env}).ExpandRelease(env.Env, "/committed", "curl.spec") + + assert.Equal(t, phase != missingTool, called) + + switch phase { + case missingTool: + require.ErrorContains(t, err, "requires host program") + case "query failure": + require.ErrorIs(t, err, queryErr) + case "missing release": + require.ErrorContains(t, err, "no Release field") + default: + require.NoError(t, err) + assert.Equal(t, "9.fc43", out) + } + }) + } +} + +func TestHostRenderSpecFiles(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RegisterCommandInSearchPath("spectool") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, "/working", cmd.Dir) + assert.Equal(t, []string{ + "spectool", "--define", "_sourcedir /working", "-l", "-a", "curl.spec", + }, cmd.Args) + + return "Source0: https://example.com/curl-1.tar.gz\nPatch0: fixes/fix.patch\n", nil + } + files, err := (hostRenderProcessor{env: env.Env}).specFiles(env.Env, "/working", "curl.spec") + require.NoError(t, err) + assert.Equal(t, []string{"curl-1.tar.gz", "fixes/fix.patch", "changelog"}, files) +} + +func TestMixedRenderRoutesOnlyLockedComponentsThroughMock(t *testing.T) { + const ( + hostFailure = "host failure" + mockFailure = "mock failure" + skipFilter = "skip host filter" + ) + + for _, phase := range []string{"success", hostFailure, mockFailure, skipFilter} { + t.Run(phase, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RegisterCommandInSearchPath("spectool") + env.CmdFactory.RegisterCommandInSearchPath("mock") + + ctrl := gomock.NewController(t) + locked := components_testutils.NewMockComponent(ctrl) + locked.EXPECT().GetName().AnyTimes().Return("curl") + + host := components_testutils.NewMockComponent(ctrl) + host.EXPECT().GetName().AnyTimes().Return("rust-asyncband") + host.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Render: projectconfig.ComponentRenderConfig{SkipFileFilter: phase == skipFilter}, + }) + + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + processor, err := renderMockProcessor(env.Env, []components.Component{host, locked}) + require.NoError(t, err) + require.NotNil(t, processor) + + queryErr := errors.New("host query failed") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.NotEqual(t, skipFilter, phase) + assert.Equal(t, "spectool", cmd.Args[0]) + assert.Equal(t, "/stage/rust-asyncband", cmd.Dir) + + if phase == hostFailure { + return "", queryErr + } + + return "Patch0: host.patch\n", nil + } + batched := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + assert.Equal(t, "mock", cmd.Args[0]) + + if phase == mockFailure { + return errors.New("mock unavailable") + } + + if strings.Contains(strings.Join(cmd.Args, " "), "render_process.py") { + batched = true + inputs, err := fileutils.ReadFile(env.TestFS, "/stage/inputs.json") + require.NoError(t, err) + assert.JSONEq(t, `[{"name":"curl","specFilename":"curl.spec"}]`, string(inputs)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/stage/results.json", + []byte(`[{"name":"curl","specFiles":"Patch0: locked.patch\n","error":null}]`), + fileperms.PublicFile)) + } + + return nil + } + results := processRenderSources(env.Env, processor, + "/stage", []*preparedComponent{ + {comp: locked, specFilename: "curl.spec"}, + {comp: host, specFilename: "rust-asyncband.spec", lockless: true}, + }) + require.Contains(t, results, "rust-asyncband") + assert.Equal(t, phase != mockFailure, batched) + + if phase == hostFailure { + require.ErrorIs(t, results["rust-asyncband"].Error, queryErr) + } else { + require.NoError(t, results["rust-asyncband"].Error) + + if phase != skipFilter { + assert.Equal(t, []string{"host.patch", "changelog"}, results["rust-asyncband"].SpecFiles) + } + } + + if phase == mockFailure { + assert.NotContains(t, results, "curl") + } else { + require.Contains(t, results, "curl") + assert.Equal(t, []string{"locked.patch"}, results["curl"].SpecFiles) + } + }) + } +} diff --git a/internal/app/azldev/cmds/component/render_modes_internal_test.go b/internal/app/azldev/cmds/component/render_modes_internal_test.go index 4bf14dfff..31147656e 100644 --- a/internal/app/azldev/cmds/component/render_modes_internal_test.go +++ b/internal/app/azldev/cmds/component/render_modes_internal_test.go @@ -6,6 +6,7 @@ package component import ( "context" "errors" + "os/exec" "path/filepath" "strings" "sync" @@ -72,16 +73,96 @@ type renderModeObservingPreparer struct { sources.SourcePreparer observe func(components.Component) err error + prepare func(components.Component, string) error } func (p *renderModeObservingPreparer) PrepareSources( - _ context.Context, component components.Component, _ string, _ bool, + _ context.Context, component components.Component, dir string, _ bool, ) error { p.observe(component) + if p.prepare != nil { + return p.prepare(component, dir) + } + return p.err } +func TestLocklessRenderPreservesMacrosWithoutMock(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + version := env.Config.Distros["test-distro"].Versions["1.0"] + version.MockConfigPath = "" + env.Config.Distros["test-distro"].Versions["1.0"] = version + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + } + env.CmdFactory.RegisterCommandInSearchPath("spectool") + + queries := 0 + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + queries++ + + assert.Equal(t, "spectool", cmd.Args[0]) + + return "Patch0: fix.patch\n", nil + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("unexpected command during host-only rendering: %v", cmd.Args) + + return nil + } + + const ( + rawSpec = "Name: curl\nVersion: 1\nRelease: %autorelease\n\n%changelog\n%autochangelog\n" + changelog = "* Old entry\n- Keep unchanged\n" + ) + + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return &renderModeObservingPreparer{ + observe: func(_ components.Component) {}, + prepare: func(_ components.Component, dir string) error { + for name, content := range map[string]string{ + "curl.spec": rawSpec, "changelog": changelog, "fix.patch": "patch", + "unreferenced": "discard", ".git/HEAD": "temporary history", + } { + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, name), + []byte(content), fileperms.PublicFile)) + } + + return nil + }, + }, nil + } + + t.Cleanup(func() { newSourcePreparer = original }) + + results, err := RenderComponents(env.Env, &RenderOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + OutputDir: "/rendered", OutputDirExplicit: true, FailOnError: true, + }) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, renderStatusOK, results[0].Status) + assert.Equal(t, 1, queries) + + for name, want := range map[string]string{"curl.spec": rawSpec, "changelog": changelog, "fix.patch": "patch"} { + content, err := fileutils.ReadFile(env.TestFS, filepath.Join("/rendered/c/curl", name)) + require.NoError(t, err) + assert.Equal(t, want, string(content)) + } + + for _, name := range []string{".git", "unreferenced"} { + exists, err := fileutils.Exists(env.TestFS, filepath.Join("/rendered/c/curl", name)) + require.NoError(t, err) + assert.False(t, exists) + } +} + func TestRenderComponentsSelectsLockModeFromFile(t *testing.T) { for _, testCase := range []struct { name string diff --git a/internal/app/azldev/core/sources/mockprocessor.go b/internal/app/azldev/core/sources/mockprocessor.go index ddfbce97e..a10ff7107 100644 --- a/internal/app/azldev/core/sources/mockprocessor.go +++ b/internal/app/azldev/core/sources/mockprocessor.go @@ -362,59 +362,6 @@ func (p *MockProcessor) runBatchScript( // the caller parses and validates it. The chroot is lazily initialized on first // use, shared with [MockProcessor.BatchProcess]. func (p *MockProcessor) CalculateRelease(ctx context.Context, specHostDir, specFilename string) (string, error) { - return p.runRPMAutospec(ctx, specHostDir, specFilename, "calculate-release", "--complete-release") -} - -// CalculateBaseRelease returns the numeric upstream release for an autorelease base. -func (p *MockProcessor) CalculateBaseRelease(ctx context.Context, specHostDir, specFilename string) (string, error) { - return p.runRPMAutospec(ctx, specHostDir, specFilename, "calculate-release", "--number-only") -} - -// GenerateChangelog returns the full changelog from the checkout's upstream history. -func (p *MockProcessor) GenerateChangelog(ctx context.Context, specHostDir, specFilename string) (string, error) { - return p.runRPMAutospec(ctx, specHostDir, specFilename, "generate-changelog") -} - -// ExpandRelease queries the spec's macro-expanded Release inside the shared mock chroot. -func (p *MockProcessor) ExpandRelease(ctx context.Context, specHostDir, specFilename string) (string, error) { - p.mu.Lock() - defer p.mu.Unlock() - - if err := p.initOnce(ctx); err != nil { - return "", err - } - - runner := p.runner.Clone() - runner.WithUnprivileged() - runner.AddBindMount(specHostDir, chrootProvenanceDir) - - cmd, err := runner.CmdInChroot(ctx, []string{ - "rpmspec", "-q", "--srpm", "--queryformat", "release=%{RELEASE}\n", - "--define", "_sourcedir " + chrootProvenanceDir, - "--define", "_specdir " + chrootProvenanceDir, - filepath.Join(chrootProvenanceDir, specFilename), - }, false) - if err != nil { - return "", fmt.Errorf("creating committed Release query in mock chroot:\n%w", err) - } - - out, err := cmd.RunAndGetOutput(ctx) - if err != nil { - return "", fmt.Errorf("querying committed Release in mock chroot:\n%w", err) - } - - for _, line := range strings.Split(out, "\n") { - if release, found := strings.CutPrefix(strings.TrimSpace(line), "release="); found && release != "" { - return release, nil - } - } - - return "", fmt.Errorf("no Release field in committed spec query output %#q", out) -} - -func (p *MockProcessor) runRPMAutospec( - ctx context.Context, specHostDir, specFilename string, args ...string, -) (string, error) { p.mu.Lock() defer p.mu.Unlock() @@ -436,7 +383,7 @@ func (p *MockProcessor) runRPMAutospec( // on the bind-mounted .git, mirroring the render batch path. Wrapped in // `sh -c` so the shell operators survive shellquote joining in CmdInChroot. shellCmd := "git config --global --add safe.directory '*' && " + - shellquote.Join(append(append([]string{"rpmautospec"}, args...), specInChroot)...) + "rpmautospec calculate-release --complete-release " + shellquote.Join(specInChroot) cmd, err := runner.CmdInChroot(ctx, []string{"sh", "-c", shellCmd}, false) if err != nil { @@ -445,7 +392,7 @@ func (p *MockProcessor) runRPMAutospec( out, err := cmd.RunAndGetOutput(ctx) if err != nil { - return "", fmt.Errorf("rpmautospec %s failed in mock chroot:\n%w", args[0], err) + return "", fmt.Errorf("rpmautospec calculate-release failed in mock chroot:\n%w", err) } return out, nil diff --git a/internal/app/azldev/core/sources/mockprocessor_test.go b/internal/app/azldev/core/sources/mockprocessor_test.go index 099a99cd6..c378a7e3f 100644 --- a/internal/app/azldev/core/sources/mockprocessor_test.go +++ b/internal/app/azldev/core/sources/mockprocessor_test.go @@ -10,7 +10,6 @@ import ( "strings" "testing" - "github.com/kballard/go-shellquote" "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" "github.com/stretchr/testify/assert" @@ -232,93 +231,6 @@ func TestCalculateRelease_Success(t *testing.T) { "git safe.directory must be configured before rpmautospec reads the .git history") } -func TestRenderRPMAutospecCommands(t *testing.T) { - for _, operation := range []string{"generate-changelog", "calculate-release --number-only"} { - t.Run(operation, func(t *testing.T) { - ctx := newMockProcessorTestCtx() - ctx.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { - assert.Contains(t, strings.Join(cmd.Args, " "), - "rpmautospec "+operation+" /tmp/provenance/curl.spec") - - return "unchanged output\n", nil - } - processor := NewMockProcessor(ctx, "/mock/config") - - var ( - out string - err error - ) - - if operation == "generate-changelog" { - out, err = processor.GenerateChangelog(ctx, "/working", "curl.spec") - } else { - out, err = processor.CalculateBaseRelease(ctx, "/working", "curl.spec") - } - - require.NoError(t, err) - assert.Equal(t, "unchanged output\n", out) - }) - } -} - -func TestExpandRelease(t *testing.T) { - const initFailure = "initialization failure" - for _, phase := range []string{"success", "diagnostics", "missing release", initFailure, "query failure"} { - t.Run(phase, func(t *testing.T) { - ctx := newMockProcessorTestCtx() - ctx.CmdFactory.RunHandler = func(_ *exec.Cmd) error { - if phase == initFailure { - return errInitBoom - } - - return nil - } - queried := false - ctx.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { - queried = true - joined := strings.Join(cmd.Args, " ") - assert.Contains(t, joined, "--unpriv") - assert.Contains(t, joined, `bind_mount:dirs=[("/committed", "/tmp/provenance")]`) - assert.Contains(t, joined, "rpmspec -q --srpm --queryformat") - - args, err := shellquote.Split(cmd.Args[len(cmd.Args)-1]) - require.NoError(t, err) - assert.Contains(t, args, "release=%{RELEASE}\n") - assert.Contains(t, joined, "_sourcedir /tmp/provenance") - assert.Contains(t, joined, "/tmp/provenance/curl.spec") - - switch phase { - case "query failure": - return "", errRpmautospecExit - case "diagnostics": - return "error: %changelog not in descending chronological order\n" + - "release=9.fc43\nwarning: trailing diagnostic\n", nil - case "missing release": - return "error: %changelog not in descending chronological order\nrelease=\n", nil - default: - return "release=9.fc43\n", nil - } - } - processor := NewMockProcessor(ctx, "/mock/config") - out, err := processor.ExpandRelease(ctx, "/committed", "curl.spec") - - assert.Equal(t, phase != initFailure, queried) - - switch phase { - case initFailure: - require.ErrorIs(t, err, errInitBoom) - case "query failure": - require.ErrorIs(t, err, errRpmautospecExit) - case "missing release": - require.ErrorContains(t, err, "no Release field") - default: - require.NoError(t, err) - assert.Equal(t, "9.fc43", out) - } - }) - } -} - func TestCalculateRelease_InitFailurePropagates(t *testing.T) { t.Parallel() diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index e078232c1..93baa850c 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -161,9 +161,9 @@ func WithUpstreamProvenance(distTag string) PreparerOption { // WithMockProcessor returns a [PreparerOption] that supplies the mock chroot // used to resolve an rpmautospec %autorelease when emitting -// %fedora_upstream_release. Without it, upstream components whose Release uses -// %autorelease get no release macro (best-effort). Running rpmautospec inside -// mock keeps it out of azldev's host dependencies. A nil processor is ignored. +// %fedora_upstream_release. Without a resolver, upstream components whose Release +// uses %autorelease get no release macro (best-effort). Lockless rendering uses +// [WithAutoreleaseResolver] instead to run on the host. A nil processor is ignored. func WithMockProcessor(mp *MockProcessor) PreparerOption { return func(p *sourcePreparerImpl) { if mp != nil { @@ -172,6 +172,13 @@ func WithMockProcessor(mp *MockProcessor) PreparerOption { } } +// WithAutoreleaseResolver selects how upstream provenance release numbers are calculated. +func WithAutoreleaseResolver(resolver AutoreleaseResolver) PreparerOption { + return func(p *sourcePreparerImpl) { + p.autoreleaseResolver = resolver + } +} + // WithAllowNoHashes returns a [PreparerOption] that allows source file // references to omit their [projectconfig.SourceFileReference.Hash] value. // When set, any source file that lacks a hash will have its hash computed @@ -238,11 +245,11 @@ type sourcePreparerImpl struct { // [WithUpstreamProvenance]. upstreamDistTag string - // autoreleaseResolver runs rpmautospec inside a mock chroot to resolve an + // autoreleaseResolver runs rpmautospec to resolve an // %autorelease Release into a concrete Fedora release number for // %fedora_upstream_release. Nil disables autorelease resolution (the release - // macro is skipped for such specs). Set via [WithMockProcessor]. - autoreleaseResolver autoreleaseResolver + // macro is skipped for such specs). Set via [WithMockProcessor] or [WithAutoreleaseResolver]. + autoreleaseResolver AutoreleaseResolver releaseStrategy releaseStrategy bumpspecCtx opctx.Ctx diff --git a/internal/app/azldev/core/sources/upstream_provenance.go b/internal/app/azldev/core/sources/upstream_provenance.go index aa8b55578..260293582 100644 --- a/internal/app/azldev/core/sources/upstream_provenance.go +++ b/internal/app/azldev/core/sources/upstream_provenance.go @@ -35,11 +35,10 @@ const distPlaceholder = "%{?dist}" // against the Azure Linux buildroot's dist tag at build time. const distPlaceholderPlain = "%{dist}" -// autoreleaseResolver resolves an rpmautospec %autorelease to a concrete Fedora -// release number by running rpmautospec in an isolated environment (a mock -// chroot in production). Abstracted so tests can substitute a fake without a -// real mock. Implemented by [*MockProcessor]. -type autoreleaseResolver interface { +// AutoreleaseResolver resolves an rpmautospec %autorelease to a concrete Fedora +// release number. Lockless rendering runs it on the host; other workflows use +// [MockProcessor]. +type AutoreleaseResolver interface { // CalculateRelease returns the raw `rpmautospec calculate-release` output // for the spec named specFilename inside specHostDir (which must hold the // pristine .git history). The caller parses and validates the result. @@ -185,8 +184,8 @@ func replaceDistTag(release, distTag string) string { } // calculateAutorelease resolves an rpmautospec %autorelease to the pristine -// Fedora release number (without the dist tag) by running rpmautospec inside a -// mock chroot via the configured [autoreleaseResolver]. Best-effort: returns "" +// Fedora release number (without the dist tag) through the configured +// [AutoreleaseResolver]. Best-effort: returns "" // with a warning when the dist-git history is unavailable, no resolver is // configured, the command fails, or the output is not a usable release number. func (p *sourcePreparerImpl) calculateAutorelease( @@ -202,7 +201,7 @@ func (p *sourcePreparerImpl) calculateAutorelease( } if p.autoreleaseResolver == nil { - slog.Warn("Skipping %fedora_upstream_release; no mock processor available to resolve %autorelease", + slog.Warn("Skipping %fedora_upstream_release; no resolver available to resolve %autorelease", "component", componentName) return "" From 9dcfc80470678aa9ff5ad9bf8a1f6e59c0d96b89 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 19:03:01 +0000 Subject: [PATCH 07/70] fix(component): skip synthetic history during lockless rendering Preserve upstream Git history for queries without generating synthetic commits or applying their legacy release adjustments during lockless rendering. This prevents a second static release bump after the explicit lockless lifecycle. Locked rendering and other workflows are unchanged. Strip leading and trailing ## comment lines from processed autorelease changelogs before carrying them into the changelog sidecar. Preserve interior comments, whitespace, and original line endings. Add regression tests and update user documentation and emitted agent skills for both changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3497f2b2-c7fb-456e-a58f-6889b6ec67e2 --- docs/user/how-to/preview-without-lockfile.md | 24 ++--- .../azldev/agentskill/content/azldev.md.tmpl | 3 +- .../agentskill/content/comp-toml.md.tmpl | 3 +- .../content/refresh-upstream-commit.md.tmpl | 8 +- .../content/withoutlockfile/azldev.md.tmpl | 3 +- .../content/withoutlockfile/comp-toml.md.tmpl | 3 +- internal/app/azldev/cmds/component/render.go | 7 +- .../azldev/cmds/component/render_lockless.go | 31 ++++++ .../render_lockless_internal_test.go | 97 ++++++++++++++++++- .../app/azldev/core/sources/sourceprep.go | 15 ++- 10 files changed, 171 insertions(+), 23 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index b3c0871d2..908d823a6 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -27,8 +27,9 @@ rust-asyncband ``` Rendering, builds, and source preparation ignore existing locks and lock -validation for listed components. They use configured upstream commits, with -synthetic history derived from committed upstream-commit TOML changes. +validation for listed components. They use configured upstream commits. +Lockless rendering does not generate synthetic history; builds and source +preparation still derive it from committed upstream-commit TOML changes. Unlisted components use lock files, lock validation, and fingerprint-derived synthetic history. Both kinds can be processed together. @@ -55,7 +56,7 @@ the current checkout's list determines which comparison to use. | Inspecting resolved state | `azldev component history`, `azldev component query` | read the generated TOML; no equivalent commands | | Lock consistency checks | On, with `--skip-lock-validation` to opt out | Skipped | | `component changed` | Compares stored input fingerprints | Compares project configuration resolved at each ref | -| Synthetic dist-git history | Derived from lock-file fingerprint changes | Derived from generated upstream-commit TOML changes | +| Synthetic dist-git history | Derived from lock-file fingerprint changes | Skipped during rendering; other workflows derive it from generated upstream-commit TOML changes | | Agent skills and MCP tools | Both workflows are available | Both workflows are available | ## Configure the Project @@ -102,11 +103,9 @@ resolved components are created or updated before the command exits with an error. Failed components remain unchanged, and orphan pruning is skipped for that run. -Commit the refreshed TOML before rendering: synthetic dist-git -history — and therefore `%autorelease` and `%autochangelog` expansion — is derived -from committed changes to the generated file. Unlike the default mode, there is no -fingerprint to compare the working tree against, so uncommitted changes do not -produce a synthetic commit. +Commit the refreshed TOML before rendering: rendering compares the upstream pins +from committed configuration. Lockless rendering neither generates synthetic +history nor expands `%autorelease` or `%autochangelog`. ## Render and Commit an Upstream Component @@ -133,7 +132,9 @@ inherits its base from the committed rendered spec. The renderer expands the committed `Release:` with host `rpmspec`, using a temporary copy of the committed dist-git files, and uses its leading integer as `-b` (for example, `9.fc43` becomes `-b 9`). It also replaces the working checkout's `changelog` -file with the committed `%changelog` body, excluding the section header. +file with the committed `%changelog` body, excluding the section header and +leading or trailing comment lines starting with `##`. Interior comments and +changelog whitespace are preserved. The later version-change release reset still applies. The last commit touching the component output is the **last render commit**. @@ -150,8 +151,9 @@ version (or there is no previous output), it restores the original upstream `updated ` followed by upstream changes on separate lines. The tool's no-op for `%autorelease` specs is accepted. -Existing overlays, synthetic history, and sidecar filtering run after these -steps. Host `spectool` supplies the sidecar keep-list, with `changelog` always +Existing overlays and sidecar filtering run after these steps. Synthetic history +and its legacy release adjustment are skipped. Host `spectool` supplies the +sidecar keep-list, with `changelog` always preserved. The spec is **not** preprocessed with `rpmautospec process-distgit`: `%autorelease`, `%autochangelog`, and other RPM macros remain unexpanded. No mock configuration is needed for lockless rendering. diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 747269ca9..a9d36acfd 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -40,7 +40,8 @@ generated upstream-commit TOML instead. All others use locks. For listed upstream components, rendering carries release/changelog state from the committed output, runs host `rpmdev-bumpspec`, then applies the existing -overlays, synthetic history, and host `spectool` filtering. Lockless rendering +overlays and host `spectool` filtering, without synthetic history or its legacy +release adjustment. Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. It preserves the `changelog` sidecar and replaces and automatically commits only that component's output directory; unrelated staged diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 05d42ecb8..35cf0380c 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -94,7 +94,8 @@ synthetic history still determine the number and order of bumps. Without the flag, no bumpspec runtime discovery occurs and the legacy static calculation is used outside the lockless upstream render workflow. That workflow always runs host `rpmdev-bumpspec -c` before existing processing, accepts its `%autorelease` -no-op, and automatically commits the component output. Commit input TOML first. +no-op, and automatically commits the component output. Lockless rendering skips +synthetic history and its legacy release adjustment. Commit input TOML first. ## Render configuration diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 458573c74..d6d1c285b 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -63,9 +63,11 @@ changelog and calculates its base release on the host. For existing output marked by a line starting with `## RPMAUTOSPEC:` and containing `autorelease`, an `%autorelease` upstream spec inherits the leading integer of the committed, macro-expanded Release as `-b` and its committed `%changelog` body as the -working `changelog` file. Expansion uses the committed files on the host; -the later version-change release reset still applies. Existing overlays, -synthetic history, and host `spectool` filtering still run afterward. +working `changelog` file, removing leading and trailing comment lines starting +with `##` while preserving interior comments. Expansion uses the committed files on the host; +the later version-change release reset still applies. Existing overlays +and host `spectool` filtering still run afterward. Synthetic history and its +legacy release adjustment are skipped during lockless rendering. Lockless rendering never uses mock or `rpmautospec process-distgit`; the output retains `%autorelease`, `%autochangelog`, and other RPM macros unexpanded, and preserves the `changelog` sidecar. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index ab4604a86..28a9ad910 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -39,7 +39,8 @@ It replaces and automatically commits only the component output directory. Commit refreshed TOML before rendering; configure Git author identity and install `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros -remain unexpanded and the `changelog` sidecar is preserved. +remain unexpanded and the `changelog` sidecar is preserved. Synthetic history +and its legacy release adjustment are skipped during rendering. `--check-only` never creates project commits. Local components skip upstream carryover and automatic commits. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index d60727f07..e8ba1be28 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -91,7 +91,8 @@ processing and automatically commits the component output directory. Commit input TOML first, install host `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed, and configure Git author identity. Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded and the -`changelog` sidecar is preserved. +`changelog` sidecar is preserved. Synthetic history and its legacy release +adjustment are skipped during rendering. `--check-only` never creates project commits. Local components skip upstream carryover and automatic commits. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 3820cd9f4..a34b1b98c 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -562,8 +562,8 @@ func prepareComponentSources( } // Prepare sources with overlays, skipping lookaside downloads. - // WithGitRepo preserves upstream .git and creates synthetic history so - // rpmautospec can expand %autorelease and %autochangelog correctly. + // Preserve upstream .git for release queries. Only locked rendering creates + // synthetic history for rpmautospec expansion. // WithSkipLookaside avoids expensive tarball downloads — only spec + // sidecar files are needed for rendering. preparerOpts := append(gitRepoPreparerOptionsForMode(env, distro, lockless), @@ -571,7 +571,8 @@ func prepareComponentSources( sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), ) if lockless { - preparerOpts = append(preparerOpts, sources.WithAutoreleaseResolver(hostRenderProcessor{env: env})) + preparerOpts = append(preparerOpts, sources.WithoutSyntheticHistory(), + sources.WithAutoreleaseResolver(hostRenderProcessor{env: env})) } else { preparerOpts = append(preparerOpts, sources.WithMockProcessor(mockProcessor)) } diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 1346ccf1e..2f47807df 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -350,6 +350,8 @@ func carryRenderedAutorelease( } _, changelog, _ := strings.Cut(string(previous[start:end]), "\n") + + changelog = stripChangelogBoundaryComments(changelog) if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "changelog"), []byte(changelog), fileperms.PublicFile); err != nil { return "", fmt.Errorf("writing committed 'changelog' content:\n%w", err) @@ -358,6 +360,35 @@ func carryRenderedAutorelease( return release, nil } +func stripChangelogBoundaryComments(changelog string) string { + lines := strings.SplitAfter(changelog, "\n") + for idx, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + + if !strings.HasPrefix(line, "##") { + break + } + + lines[idx] = "" + } + + for idx := len(lines) - 1; idx >= 0; idx-- { + if strings.TrimSpace(lines[idx]) == "" { + continue + } + + if !strings.HasPrefix(lines[idx], "##") { + break + } + + lines[idx] = "" + } + + return strings.Join(lines, "") +} + func expandCommittedRelease( env *azldev.Env, autospec renderAutorelease, dir, filename string, history lastRender, ) (release string, err error) { diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 1025b6b73..a5f04160c 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -13,6 +13,7 @@ import ( "strings" "testing" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" @@ -275,7 +276,9 @@ func testCommittedAutoreleaseHistory(t *testing.T) lastRender { t.Helper() content := "## RPMAUTOSPEC: processed with autorelease, autochangelog\n%global carried_release 9\n" + - renderTestSpec("1", "%{carried_release}%{?dist}", carriedAutoreleaseChangelog) + + renderTestSpec("1", "%{carried_release}%{?dist}", + "## START: rpmautospec generated changelog\n"+carriedAutoreleaseChangelog+ + "## END: rpmautospec generated changelog\n") + "%files\n/file\n" repo, commits := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ "curl.spec": []byte(content), "macros.inc": []byte("committed macro definitions"), @@ -286,6 +289,33 @@ func testCommittedAutoreleaseHistory(t *testing.T) lastRender { return lastRender{spec: content, tree: tree, previous: "old", current: "old"} } +func TestStripChangelogBoundaryComments(t *testing.T) { + for _, testCase := range []struct { + name, input, want string + }{ + {name: "empty"}, + {name: "unchanged", input: "* Entry\n- Keep spaces \n\n", want: "* Entry\n- Keep spaces \n\n"}, + {name: "leading only", input: "## First\n## Second\n* Entry\n", want: "* Entry\n"}, + {name: "trailing only", input: "* Entry\n## First\n## Second", want: "* Entry\n"}, + { + name: "boundary blocks and interior comments", + input: "\n## Start\n\n* Entry\n## Interior\n- Keep\n\n## End\n\n", + want: "\n\n* Entry\n## Interior\n- Keep\n\n\n", + }, + {name: "comments only", input: "## Start\n## End\n", want: ""}, + {name: "CRLF", input: "## Start\r\n* Entry\r\n## End\r\n", want: "* Entry\r\n"}, + { + name: "other comments remain", + input: "# Start\n## Interior\n* Entry\n ## Indented\n", + want: "# Start\n## Interior\n* Entry\n ## Indented\n", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + assert.Equal(t, testCase.want, stripChangelogBoundaryComments(testCase.input)) + }) + } +} + func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { for _, testCase := range []struct { name string @@ -478,6 +508,71 @@ func TestPrepareComponentSources_LocalAndLockedSkipNewLifecycle(t *testing.T) { } } +func TestPrepareComponentSources_LocklessSkipsSyntheticHistory(t *testing.T) { + for _, release := range []string{"7%{?dist}", "%autorelease -b 9"} { + t.Run(release, func(t *testing.T) { + env := testutils.NewTestEnv(t) + ctrl := gomock.NewController(t) + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + }) + + manager := sourceproviders_test.NewMockSourceManager(ctrl) + + const changelog = "* Existing entry\n- Keep unchanged\n" + + manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string, opts ...sourceproviders.FetchComponentOption) error { + var resolved sourceproviders.FetchComponentOptions + for _, opt := range opts { + opt(&resolved) + } + + assert.True(t, resolved.PreserveGitDir) + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(dir, ".git/HEAD"), + []byte("upstream history"), fileperms.PublicFile)) + + return fileutils.WriteFile(env.TestFS, filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("1", release, changelog)), fileperms.PublicFile) + }) + + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + t.Cleanup(func() { newSourcePreparer = original }) + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("unexpected synthetic-history command: %v", cmd.Args) + + return nil + } + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected synthetic-history query: %v", cmd.Args) + + return "", nil + } + + _, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/rendered", + spec.EditorLegacy, false, true) + require.NoError(t, err) + content, err := fileutils.ReadFile(env.TestFS, "/staging/curl/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: "+release+"\n") + assert.Contains(t, string(content), "%changelog\n"+changelog) + content, err = fileutils.ReadFile(env.TestFS, "/staging/curl/.git/HEAD") + require.NoError(t, err) + assert.Equal(t, "upstream history", string(content)) + }) + } +} + func TestPrepareLocklessSpec_StopsOnAutospecFailure(t *testing.T) { env := testutils.NewTestEnv(t) require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 93baa850c..1ed6af893 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -122,6 +122,14 @@ func WithoutLockfileHistory() PreparerOption { } } +// WithoutSyntheticHistory preserves the upstream Git repository without generating +// synthetic commits or applying their release adjustments. +func WithoutSyntheticHistory() PreparerOption { + return func(p *sourcePreparerImpl) { + p.skipSyntheticHistory = true + } +} + // WithSpecEditor selects the [spec.EditorMode] used for source preparation. func WithSpecEditor(mode spec.EditorMode) PreparerOption { return func(p *sourcePreparerImpl) { p.specEditor = mode } @@ -208,7 +216,8 @@ type sourcePreparerImpl struct { // withGitRepo, when true, enables dist-git creation by preserving the // upstream .git directory and generating synthetic commit history. - withGitRepo bool + withGitRepo bool + skipSyntheticHistory bool // skipLookaside, when true, skips all lookaside cache downloads during // source preparation. Git-tracked files are still fetched. @@ -573,6 +582,10 @@ func (p *sourcePreparerImpl) trySyntheticHistory( fingerprintConfig *projectconfig.ComponentConfig, sourcesDirPath string, ) error { + if p.skipSyntheticHistory { + return nil + } + config := component.GetConfig() componentName := component.GetName() From 7d0852cff37300ab2b13df0140e6f6634b2a1333 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 19:20:34 +0000 Subject: [PATCH 08/70] fix(component): recover last render upstream pin from historical locks When the last render has no spec.upstream-commit, use its lock file only if that commit's top-level lockless-components.txt exists and does not list the component. Resolve the lock directory from the historical configuration and read only committed content from that snapshot. Keep configured pins authoritative and require spec.upstream-commit for the current render. Missing or invalid historical locks retain the existing unknown-upstream fallback. Add regression tests for fallback eligibility, historical lock directory resolution, committed content isolation, and current-pin requirements. Update user documentation and emitted agent guidance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3497f2b2-c7fb-456e-a58f-6889b6ec67e2 --- docs/user/how-to/preview-without-lockfile.md | 5 + .../content/refresh-upstream-commit.md.tmpl | 5 + .../azldev/cmds/component/render_lockless.go | 52 ++++++++++- .../render_lockless_internal_test.go | 91 +++++++++++++++++++ 4 files changed, 150 insertions(+), 3 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 908d823a6..7b6bed32b 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -140,6 +140,11 @@ The later version-change release reset still applies. The last commit touching the component output is the **last render commit**. The renderer fully loads TOML, including includes and component defaults, at that commit and at the current project commit to compare `spec.upstream-commit`. +If the last render has no configured pin, its committed lock file supplies +`upstream-commit` only when that commit's top-level `lockless-components.txt` +exists and does not list the component. The historical configuration's +`lock-dir` determines which lock file to read; current or uncommitted locks are +never used for this fallback. Different pins, missing render history, or an error loading the old pin mean upstream changed. It records `git log --oneline ..` from the upstream checkout; without an old pin it records a one-line version update. diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index d6d1c285b..56c2b8bd7 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -72,6 +72,11 @@ Lockless rendering never uses mock or `rpmautospec process-distgit`; the output retains `%autorelease`, `%autochangelog`, and other RPM macros unexpanded, and preserves the `changelog` sidecar. +When the last render's configuration has no `spec.upstream-commit`, its +committed lock file can supply the old pin only if that commit's top-level +`lockless-components.txt` exists and does not list the component. The current +render still requires a configured pin. + Rendering then replaces `{{ .RenderedSpecsDir }}///` and automatically commits only that directory. The subject is `updated ` and the body contains upstream changes, if any. diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 2f47807df..b83a0ece1 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -22,6 +22,8 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/lockfile" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" @@ -151,14 +153,14 @@ func renderedUpstreamCommits( return "", "", err } - current, err = renderUpstreamCommit(env, tree, projectPath, name) + current, err = renderUpstreamCommit(env, tree, projectPath, name, false) if err != nil { return "", "", fmt.Errorf("reading current render upstream commit:\n%w", err) } lastTree, err := last.Tree() if err == nil { - previous, err = renderUpstreamCommit(env, lastTree, projectPath, name) + previous, err = renderUpstreamCommit(env, lastTree, projectPath, name, true) } if err != nil { @@ -171,7 +173,9 @@ func renderedUpstreamCommits( return previous, current, nil } -func renderUpstreamCommit(env *azldev.Env, tree *object.Tree, projectPath, name string) (string, error) { +func renderUpstreamCommit( + env *azldev.Env, tree *object.Tree, projectPath, name string, allowLockFallback bool, +) (string, error) { historicalEnv, err := loadHistoricalEnv(env, tree, projectPath) if err != nil { return "", err @@ -183,6 +187,10 @@ func renderUpstreamCommit(env *azldev.Env, tree *object.Tree, projectPath, name } commit := comp.GetConfig().Spec.UpstreamCommit + if commit == "" && allowLockFallback { + return lastRenderLockedUpstreamCommit(historicalEnv, name) + } + if commit == "" { return "", fmt.Errorf("component %#q has no configured 'spec.upstream-commit'", name) } @@ -190,6 +198,44 @@ func renderUpstreamCommit(env *azldev.Env, tree *object.Tree, projectPath, name return commit, nil } +func lastRenderLockedUpstreamCommit(env *azldev.Env, name string) (string, error) { + selectionPath := filepath.Join(env.ProjectDir(), "lockless-components.txt") + + exists, err := fileutils.Exists(env.FS(), selectionPath) + if err != nil { + return "", fmt.Errorf("checking historical lockless component selection:\n%w", err) + } + + if !exists { + return "", errors.New("no historical 'lockless-components.txt' for lock fallback") + } + + selected, err := projectconfig.ReadLocklessComponents(env.FS(), env.ProjectDir()) + if err != nil { + return "", fmt.Errorf("reading historical lockless component selection:\n%w", err) + } + + if selected[name] { + return "", fmt.Errorf("historical component %#q is lockless but has no 'spec.upstream-commit'", name) + } + + lockDir := env.Config().Project.LockDir + if lockDir == "" { + return "", errors.New("no historical 'lock-dir' configured for lock fallback") + } + + lock, err := lockfile.NewStore(env.FS(), lockDir).Get(name) + if err != nil { + return "", fmt.Errorf("reading historical lock for %#q:\n%w", name, err) + } + + if lock.UpstreamCommit == "" { + return "", fmt.Errorf("historical lock for %#q has no 'upstream-commit'", name) + } + + return lock.UpstreamCommit, nil +} + func prepareLocklessSpec( env *azldev.Env, autospec renderAutorelease, dir, name string, editor spec.EditorMode, history lastRender, update *locklessRenderUpdate, diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index a5f04160c..0471c3513 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -77,6 +77,97 @@ release-ver = "1" assert.Equal(t, "bbbbbbb", history.current) } +func TestReadLastRender_HistoricalLockFallback(t *testing.T) { + const ( + oldLock = "version = 1\nupstream-commit = 'aaaaaaa'\n" + config = `[project] +default-distro = { name = "test", version = "1" } +lock-dir = "../historical-locks" +[distros.test.versions."1"] +release-ver = "1" +` + ) + for _, testCase := range []struct { + name, selection, lock, pin, want string + noSelection, noLock, noLockDir bool + missingCurrent bool + }{ + {name: "historically locked", selection: "rust-asyncband\n", lock: oldLock, want: "aaaaaaa"}, + {name: "empty selection", lock: oldLock, want: "aaaaaaa"}, + {name: "exact names", selection: "Curl\ncurl-extra\n", lock: oldLock, want: "aaaaaaa"}, + {name: "listed component", selection: " curl \r\n", lock: oldLock}, + {name: "missing selection", noSelection: true, lock: oldLock}, + {name: "missing lock", noLock: true}, + {name: "corrupt lock", lock: "invalid = ["}, + {name: "empty lock pin", lock: "version = 1\n"}, + {name: "no configured lock directory", noLockDir: true, lock: oldLock}, + {name: "configured pin takes precedence", pin: "ccccccc", lock: oldLock, want: "ccccccc"}, + {name: "no fallback for current pin", lock: oldLock, missingCurrent: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + oldConfig := config + if testCase.noLockDir { + oldConfig = strings.ReplaceAll(oldConfig, `lock-dir = "../historical-locks"`, `lock-dir = ""`) + } + + component := "[components.curl.spec]\ntype = 'upstream'\n" + + files := map[string][]byte{ + "azldev.toml": []byte(`includes = ["settings/project.toml", "components.toml"]`), + "settings/project.toml": []byte(oldConfig), + "components.toml": []byte(component + + "upstream-commit = '" + testCase.pin + "'\n"), + "SPECS/c/curl/curl.spec": []byte(renderTestSpec("1", "9", "old\n")), + } + if !testCase.noSelection { + files["lockless-components.txt"] = []byte(testCase.selection) + } + + if !testCase.noLock { + files["historical-locks/curl.lock"] = []byte(testCase.lock) + } + + current := component + selection := "rust-asyncband\n" + + if !testCase.missingCurrent { + current += "upstream-commit = 'bbbbbbb'\n" + selection = "curl\n" + } + + repo, _ := testRepoWithCommits(t, []testRepoCommit{ + {files: files}, + {files: map[string][]byte{ + "components.toml": []byte(current), + "settings/project.toml": []byte(strings.ReplaceAll(config, "historical-locks", "current-locks")), + "lockless-components.txt": []byte(selection), + "historical-locks/curl.lock": []byte("version = 1\nupstream-commit = 'changed-old-lock'\n"), + "current-locks/curl.lock": []byte("version = 1\nupstream-commit = 'current-lock'\n"), + }}, + }) + worktree, err := repo.Worktree() + require.NoError(t, err) + dirty, err := worktree.Filesystem.Create("historical-locks/curl.lock") + require.NoError(t, err) + _, err = dirty.Write([]byte("uncommitted lock")) + require.NoError(t, err) + require.NoError(t, dirty.Close()) + env := testutils.NewTestEnv(t) + + history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl") + if testCase.missingCurrent { + require.ErrorContains(t, err, "current render upstream commit") + + return + } + + require.NoError(t, err) + assert.Equal(t, testCase.want, history.previous) + assert.Equal(t, "bbbbbbb", history.current) + }) + } +} + func TestReadLastRender_AbsentAndInvalidHistory(t *testing.T) { valid := []byte(`[project] default-distro = { name = "test", version = "1" } From ea367f330335699b87064a0aea779bedfeeea491 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 19:29:58 +0000 Subject: [PATCH 09/70] fix(component): allow historical lock fallback without selection file Use the historical lock's upstream-commit when spec.upstream-commit is missing and lockless-components.txt is absent or excludes the component. Continue rejecting the fallback for explicitly listed lockless components. Update regression tests and documentation to cover an absent selection file with and without a component lock. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3497f2b2-c7fb-456e-a58f-6889b6ec67e2 --- docs/user/how-to/preview-without-lockfile.md | 4 ++-- .../content/refresh-upstream-commit.md.tmpl | 4 ++-- internal/app/azldev/cmds/component/render_lockless.go | 11 ----------- .../cmds/component/render_lockless_internal_test.go | 3 ++- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 7b6bed32b..16520d144 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -141,8 +141,8 @@ The last commit touching the component output is the **last render commit**. The renderer fully loads TOML, including includes and component defaults, at that commit and at the current project commit to compare `spec.upstream-commit`. If the last render has no configured pin, its committed lock file supplies -`upstream-commit` only when that commit's top-level `lockless-components.txt` -exists and does not list the component. The historical configuration's +`upstream-commit` when that commit's top-level `lockless-components.txt` is absent +or does not list the component. The historical configuration's `lock-dir` determines which lock file to read; current or uncommitted locks are never used for this fallback. Different pins, missing render history, or an error loading the old pin mean diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 56c2b8bd7..d1bf19838 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -73,8 +73,8 @@ retains `%autorelease`, `%autochangelog`, and other RPM macros unexpanded, and preserves the `changelog` sidecar. When the last render's configuration has no `spec.upstream-commit`, its -committed lock file can supply the old pin only if that commit's top-level -`lockless-components.txt` exists and does not list the component. The current +committed lock file can supply the old pin if that commit's top-level +`lockless-components.txt` is absent or does not list the component. The current render still requires a configured pin. Rendering then replaces `{{ .RenderedSpecsDir }}///` and diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index b83a0ece1..139d4ca3c 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -199,17 +199,6 @@ func renderUpstreamCommit( } func lastRenderLockedUpstreamCommit(env *azldev.Env, name string) (string, error) { - selectionPath := filepath.Join(env.ProjectDir(), "lockless-components.txt") - - exists, err := fileutils.Exists(env.FS(), selectionPath) - if err != nil { - return "", fmt.Errorf("checking historical lockless component selection:\n%w", err) - } - - if !exists { - return "", errors.New("no historical 'lockless-components.txt' for lock fallback") - } - selected, err := projectconfig.ReadLocklessComponents(env.FS(), env.ProjectDir()) if err != nil { return "", fmt.Errorf("reading historical lockless component selection:\n%w", err) diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 0471c3513..eb6fe8339 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -96,7 +96,8 @@ release-ver = "1" {name: "empty selection", lock: oldLock, want: "aaaaaaa"}, {name: "exact names", selection: "Curl\ncurl-extra\n", lock: oldLock, want: "aaaaaaa"}, {name: "listed component", selection: " curl \r\n", lock: oldLock}, - {name: "missing selection", noSelection: true, lock: oldLock}, + {name: "missing selection", noSelection: true, lock: oldLock, want: "aaaaaaa"}, + {name: "missing selection and lock", noSelection: true, noLock: true}, {name: "missing lock", noLock: true}, {name: "corrupt lock", lock: "invalid = ["}, {name: "empty lock pin", lock: "version = 1\n"}, From af6c5fa85d1861fabe93dd76c90a06f69f793d45 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 20:14:09 +0000 Subject: [PATCH 10/70] fix(component): carry over baserelease definitions Carry static release state directly from the committed dist-git spec into working upstream specs. If the committed Release references baserelease, copy its complete %global or %define definition and leave the upstream Release value unchanged. Otherwise, copy the committed Release value. Reject missing or ambiguous baserelease definitions. Preserve committed changelogs and retain upstream release values when an upstream change also changes Version. Run rpmdev-bumpspec only for the final release bump and changelog update, without temporary bump probes or context-based diff matching. Cover macro and literal releases, version resets, missing or ambiguous definitions, and single-bump behavior. Update the user guide and emitted agent guidance to describe the direct carryover rules. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 15 +- .../content/refresh-upstream-commit.md.tmpl | 14 +- .../azldev/cmds/component/render_lockless.go | 43 +++-- .../render_lockless_internal_test.go | 1 + .../component/render_release_carryover.go | 91 +++++++++ .../render_release_carryover_internal_test.go | 176 ++++++++++++++++++ 6 files changed, 317 insertions(+), 23 deletions(-) create mode 100644 internal/app/azldev/cmds/component/render_release_carryover.go create mode 100644 internal/app/azldev/cmds/component/render_release_carryover_internal_test.go diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 16520d144..325cb367e 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -118,8 +118,13 @@ The renderer clones the selected upstream dist-git to temporary storage. It finds the existing committed output under `//`, ignoring uncommitted spec changes. For an existing non-`%autorelease` upstream spec, it -carries over the committed rendered spec's exact `Release:` value and -`%changelog` section. +checks the committed `Release:` value. If it references the `baserelease` macro, +the renderer copies the committed `%global baserelease` or `%define baserelease` +line over the upstream definition, leaving the upstream `Release:` value +unchanged. A missing or ambiguous macro definition fails rendering. Otherwise, +it copies the committed `Release:` value into the upstream spec's `Release:`. +No temporary release-bump probe is performed. The committed `%changelog` +section is also preserved. For an initial `%autorelease` render, it generates the complete upstream `changelog` file (replacing any existing file), calculates the numeric upstream @@ -151,8 +156,10 @@ upstream checkout; without an old pin it records a one-line version update. An error reading the current pin is fatal. When upstream changes exist and `Version:` differs from the committed rendered -version (or there is no previous output), it restores the original upstream -`Release:`. It then always invokes host `rpmdev-bumpspec -c` with +version (or there is no previous output), it retains the original upstream +`Release:` and `baserelease` definition instead of carrying over previous +values, and restores the upstream `Release:` if an autorelease base was carried over. +It then always invokes host `rpmdev-bumpspec -c` with `updated ` followed by upstream changes on separate lines. The tool's no-op for `%autorelease` specs is accepted. diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index d1bf19838..d0c0f5015 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -55,10 +55,16 @@ git commit -m "fix(): ..." azldev comp render -p ``` -Rendering preserves the committed downstream Release and changelog for static -specs, resets Release to its original upstream value on a version change, and -runs host `rpmdev-bumpspec -c` with `updated ` plus upstream -changes. An initial `%autorelease` render generates the complete upstream +For static specs, rendering checks the committed `Release:` value. If it uses +the `baserelease` macro, rendering copies the committed `%global baserelease` +or `%define baserelease` line over the upstream definition without changing +the upstream `Release:` value. Missing or ambiguous definitions fail rendering. +Otherwise, it copies the committed `Release:` value. No temporary release-bump +probe is performed, and the committed changelog is retained. A version change +keeps the original upstream `Release:` and `baserelease` definition before +the final host `rpmdev-bumpspec -c` with +`updated ` plus upstream changes. +An initial `%autorelease` render generates the complete upstream changelog and calculates its base release on the host. For existing output marked by a line starting with `## RPMAUTOSPEC:` and containing `autorelease`, an `%autorelease` upstream spec inherits the leading integer of the committed, diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 139d4ca3c..5fd401240 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -272,6 +272,10 @@ func prepareLocklessSpec( return fmt.Errorf("updating upstream Release:\n%w", err) } + if err := carryStaticRenderRelease(env, opened, original, version, history, update.changes != "", editor); err != nil { + return err + } + resultBytes, err := locklessSpecBytes(opened, history.spec, original, editor) if err != nil { return err @@ -427,12 +431,33 @@ func stripChangelogBoundaryComments(changelog string) string { func expandCommittedRelease( env *azldev.Env, autospec renderAutorelease, dir, filename string, history lastRender, ) (release string, err error) { - snapshot, err := fileutils.MkdirTemp(env.FS(), filepath.Dir(dir), "azldev-render-previous-") + snapshot, err := committedRenderSnapshot(env, dir, filename, history) if err != nil { - return "", fmt.Errorf("creating committed dist-git snapshot:\n%w", err) + return "", err } defer fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), snapshot, &err) + release, err = autospec.ExpandRelease(env, snapshot, filename) + if err != nil { + return "", fmt.Errorf("querying committed spec:\n%w", err) + } + + return release, nil +} + +func committedRenderSnapshot( + env *azldev.Env, dir, filename string, history lastRender, +) (snapshot string, err error) { + snapshot, err = fileutils.MkdirTemp(env.FS(), filepath.Dir(dir), "azldev-render-previous-") + if err != nil { + return "", fmt.Errorf("creating committed dist-git snapshot:\n%w", err) + } + defer func(path string) { + if err != nil { + fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), path, &err) + } + }(snapshot) + if history.tree != nil { if err := copyTreeToFS(history.tree, env.FS(), snapshot); err != nil { return "", fmt.Errorf("copying committed dist-git snapshot:\n%w", err) @@ -444,12 +469,7 @@ func expandCommittedRelease( return "", fmt.Errorf("writing committed spec snapshot:\n%w", err) } - release, err = autospec.ExpandRelease(env, snapshot, filename) - if err != nil { - return "", fmt.Errorf("querying committed spec:\n%w", err) - } - - return release, nil + return snapshot, nil } func restoreLocklessRelease( @@ -468,13 +488,6 @@ func restoreLocklessRelease( return "", fmt.Errorf("parsing committed dist-git spec:\n%w", err) } - if !sources.ReleaseUsesAutorelease(original) { - release, err = previous.GetLastTag("", "Release") - if err != nil { - return "", fmt.Errorf("reading committed Release:\n%w", err) - } - } - if !upstreamChanged { return release, nil } diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index eb6fe8339..0fb506d9d 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -321,6 +321,7 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { } called := false env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + assert.False(t, called, "only the final release bump should run") called = true entry := "updated curl" diff --git a/internal/app/azldev/cmds/component/render_release_carryover.go b/internal/app/azldev/cmds/component/render_release_carryover.go new file mode 100644 index 000000000..6cb3890ab --- /dev/null +++ b/internal/app/azldev/cmds/component/render_release_carryover.go @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "fmt" + "regexp" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" +) + +var ( + baseReleaseMacro = regexp.MustCompile(`%(?:\{\??baserelease\}|baserelease\b)`) + baseReleaseLine = regexp.MustCompile(`^[ \t]*%(?:global|define)[ \t]+baserelease[ \t]+\S`) +) + +func carryStaticRenderRelease( + env *azldev.Env, working *spec.Spec, release, version string, + history lastRender, upstreamChanged bool, editor spec.EditorMode, +) error { + if history.spec == "" || sources.ReleaseUsesAutorelease(release) || env.DryRun() { + return nil + } + + previous, err := spec.OpenSpec(strings.NewReader(history.spec), spec.WithEditor(editor)) + if err != nil { + return fmt.Errorf("parsing committed release spec:\n%w", err) + } + + oldVersion, err := previous.GetLastTag("", "Version") + if err != nil { + return fmt.Errorf("reading committed Version:\n%w", err) + } + + if upstreamChanged && oldVersion != version { + return nil + } + + oldRelease, err := previous.GetLastTag("", "Release") + if err != nil { + return fmt.Errorf("reading committed Release:\n%w", err) + } + + if !baseReleaseMacro.MatchString(oldRelease) { + if err := working.SetTag("", "Release", oldRelease); err != nil { + return fmt.Errorf("carrying committed Release:\n%w", err) + } + + return nil + } + + _, definition, err := baseReleaseDefinition(previous) + if err != nil { + return fmt.Errorf("reading committed 'baserelease' definition:\n%w", err) + } + + line, _, err := baseReleaseDefinition(working) + if err != nil { + return fmt.Errorf("finding upstream 'baserelease' definition:\n%w", err) + } + + working.ReplaceLine(line, definition) + + return nil +} + +func baseReleaseDefinition(opened *spec.Spec) (line int, definition string, err error) { + var matches int + + err = opened.Visit(func(ctx *spec.Context) error { + if ctx.CurrentSection.SectName == "" && ctx.RawLine != nil && baseReleaseLine.MatchString(*ctx.RawLine) { + line, definition = ctx.CurrentLineNum, *ctx.RawLine + matches++ + } + + return nil + }) + if err != nil { + return 0, "", fmt.Errorf("finding 'baserelease' macro:\n%w", err) + } + + if matches != 1 { + return 0, "", fmt.Errorf("expected one 'baserelease' definition, found %d", matches) + } + + return line, definition, nil +} diff --git a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go new file mode 100644 index 000000000..557eb093e --- /dev/null +++ b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "os/exec" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + for _, testCase := range []struct { + name, version, definition, release, wantBase, wantBumped string + changed bool + }{ + { + name: "same upstream", version: "1", definition: "%global baserelease 18", + release: "%{baserelease}%{?dist}", wantBase: "18", wantBumped: "19", + }, + { + name: "new upstream same version", version: "1", changed: true, + definition: "%global baserelease 18", release: "%{baserelease}%{?dist}", + wantBase: "18", wantBumped: "19", + }, + { + name: "new version", version: "2", changed: true, + definition: "%global baserelease 18", release: "%{baserelease}%{?dist}", + wantBase: "17", wantBumped: "18", + }, + { + name: "define and optional macro", version: "1", definition: "%define baserelease 18", + release: "%{?baserelease}%{?dist}", wantBase: "18", wantBumped: "19", + }, + { + name: "unbraced macro", version: "1", definition: "%global baserelease 18", + release: "%baserelease%{?dist}", wantBase: "18", wantBumped: "19", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + const ( + path = "/working/curl.spec" + oldChangelog = "* Old downstream entry\n" + newRelease = "%{baserelease}.upstream%{?dist}" + ) + + oldSpec := testCase.definition + "\n" + + renderTestSpec("1", testCase.release, oldChangelog) + + history := lastRender{spec: oldSpec, previous: "old", current: "old"} + if testCase.changed { + history.current = "new" + } + + // Upstream context and Release differ; only the macro definition is carried over. + upstream := "# New upstream header\n%define baserelease 17\n" + + renderTestSpec(testCase.version, newRelease, "* Upstream entry\n") + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(upstream), fileperms.PublicFile)) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return "abc123 Upstream change\n", nil + } + + var calls int + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + calls++ + + assert.Equal(t, path, cmd.Args[len(cmd.Args)-1]) + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + + definition := testCase.definition + if testCase.changed && testCase.version != "1" { + definition = "%define baserelease 17" + } + + assert.Contains(t, string(content), definition+"\n") + assert.Contains(t, string(content), "Release: "+newRelease+"\n") + assert.Contains(t, string(content), "%changelog\n"+oldChangelog) + + return fileutils.WriteFile(env.TestFS, path, + []byte(strings.Replace(string(content), "baserelease "+testCase.wantBase, + "baserelease "+testCase.wantBumped, 1)), fileperms.PublicFile) + } + err := prepareLocklessSpec(env.Env, &renderAutospecStub{}, "/working", "curl", editor, + history, &locklessRenderUpdate{subject: "updated curl"}) + require.NoError(t, err) + assert.Equal(t, 1, calls) + + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Contains(t, string(content), "baserelease "+testCase.wantBumped+"\n") + assert.Equal(t, oldSpec, history.spec) + }) + } + }) + } +} + +func TestCarryStaticRenderRelease(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + for _, testCase := range []struct { + name, previous, upstream, oldRelease, wantRelease, wantError string + }{ + { + name: "literal release with unrelated baserelease definition", + previous: "%global baserelease 18\n", upstream: "%global baserelease 17\n", + oldRelease: "9%{?dist}", wantRelease: "9%{?dist}", + }, + { + name: "similarly named macro", oldRelease: "%{baserelease_extra}", + wantRelease: "%{baserelease_extra}", + }, + { + name: "missing committed definition", upstream: "%global baserelease 17\n", + oldRelease: "%{baserelease}", wantError: "reading committed 'baserelease' definition", + }, + { + name: "missing upstream definition", previous: "%global baserelease 18\n", + oldRelease: "%{baserelease}", wantError: "finding upstream 'baserelease' definition", + }, + { + name: "ambiguous committed definition", + previous: "%global baserelease 18\n%define baserelease 19\n", + upstream: "%global baserelease 17\n", oldRelease: "%{baserelease}", + wantError: "expected one 'baserelease' definition, found 2", + }, + { + name: "ambiguous upstream definition", previous: "%global baserelease 18\n", + upstream: "%global baserelease 16\n%define baserelease 17\n", + oldRelease: "%{baserelease}", wantError: "expected one 'baserelease' definition, found 2", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + upstream := testCase.upstream + renderTestSpec("1", "1%{?dist}", "") + opened, err := spec.OpenSpec(strings.NewReader(upstream), spec.WithEditor(editor)) + require.NoError(t, err) + + history := lastRender{spec: testCase.previous + renderTestSpec("1", testCase.oldRelease, "")} + + err = carryStaticRenderRelease(env.Env, opened, "1%{?dist}", "1", history, false, editor) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + + return + } + + require.NoError(t, err) + release, err := opened.GetLastTag("", "Release") + require.NoError(t, err) + assert.Equal(t, testCase.wantRelease, release) + + if testCase.upstream != "" { + _, definition, err := baseReleaseDefinition(opened) + require.NoError(t, err) + assert.Equal(t, strings.TrimSuffix(testCase.upstream, "\n"), definition) + } + }) + } + }) + } +} From 58b09e3f1d6b441f62bc58efe5cfe2904d54702a Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 21:00:23 +0000 Subject: [PATCH 11/70] perf(component): clone only the selected upstream branch Clone the configured upstream dist-git branch with --single-branch while retaining its full history for pinned commits and snapshot resolution. Resolve the previous and current branches from committed project config, including per-component distro overrides and historical lock fallback. When differing upstream pins come from different branches, fetch the old branch into the working clone before collecting upstream changes. Keep the current checkout unchanged and propagate historical fetch failures. Add regression coverage for clone arguments, historical branch resolution, branch transitions, and failed fetches. Update user and emitted agent documentation to explain branch-limited cloning and historical fetches. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 11 +- docs/user/reference/config/components.md | 9 ++ .../app/azldev/agentskill/agentskill_test.go | 2 + .../content/refresh-upstream-commit.md.tmpl | 6 + .../azldev/cmds/component/render_lockless.go | 61 ++++++---- .../render_lockless_internal_test.go | 107 +++++++++++++++++- internal/utils/git/git.go | 5 +- internal/utils/git/git_test.go | 29 ++--- 8 files changed, 187 insertions(+), 43 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 325cb367e..27e67970d 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -114,7 +114,11 @@ lockless**. Local components skip the upstream release/changelog carryover and automatic commit steps. All lockless rendering runs on the host without mock; components using lock files retain their existing mock-based rendering. -The renderer clones the selected upstream dist-git to temporary storage. It +The renderer clones only the branch selected by the component's effective +`spec.upstream-distro` (or the project default) into temporary storage. The +resolved distro version's `dist-git-branch` supplies the branch name. The clone +retains that branch's full history, not every upstream branch, so pinned commits +and snapshot selection remain available without a shallow-history limit. It finds the existing committed output under `//`, ignoring uncommitted spec changes. For an existing non-`%autorelease` upstream spec, it @@ -150,6 +154,11 @@ If the last render has no configured pin, its committed lock file supplies or does not list the component. The historical configuration's `lock-dir` determines which lock file to read; current or uncommitted locks are never used for this fallback. +When the last render used a different upstream branch and the pins differ, the +renderer fetches that branch into the working clone before comparing commits. +Its branch is resolved from the last render's committed TOML, including includes, +component overrides, and distro defaults. The current branch remains available; +fetching the historical branch does not change the working checkout. Different pins, missing render history, or an error loading the old pin mean upstream changed. It records `git log --oneline ..` from the upstream checkout; without an old pin it records a one-line version update. diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 3fa997c9b..3d5f5dd23 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -76,6 +76,11 @@ When the upstream package has a different name than the component, use `upstream spec = { type = "upstream", upstream-name = "redhat-rpm-config" } ``` +Upstream clones fetch only the effective distro version's `dist-git-branch`, +including its full history. They are not shallow clones and do not fetch every +upstream branch. Select a different branch through `spec.upstream-distro` and +the referenced distro version's `dist-git-branch`. + ### Commit-Pinned Specs When you need to pin to a specific git commit — for example, to pick up a fix that hasn't been included in a tagged release yet — use `upstream-commit`: @@ -87,6 +92,10 @@ spec = { type = "upstream", upstream-commit = "a1b2c3d4e5f6789" } The value must be a hex string between 7 and 40 characters (a short or full git commit hash). When `upstream-commit` is set, it takes priority over the distro snapshot date — azldev checks out the exact commit instead of finding the latest commit before a snapshot timestamp. +The pinned commit must be available from the selected upstream branch. Lockless +rendering additionally fetches the last render's branch when it differs and +the old and current pins must be compared. + > **Note:** Commit-pinning is intended for temporary use. Once the desired change lands in a tagged upstream release, switch back to version-based pinning or the default snapshot to keep the component aligned with the upstream distro. ### Local Specs diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 7941b6934..9334ba3fb 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -186,6 +186,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "## RPMAUTOSPEC:") assert.Contains(t, doc, "leading integer of the committed,") assert.Contains(t, doc, "retains `%autorelease`, `%autochangelog`") + assert.Contains(t, doc, "effective upstream distro's `dist-git-branch`") + assert.Contains(t, doc, "fetches that historical branch before comparing commits") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index d0c0f5015..15f764006 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -55,6 +55,12 @@ git commit -m "fix(): ..." azldev comp render -p ``` +Rendering clones only the effective upstream distro's `dist-git-branch`, keeping +its full history rather than making a shallow clone. If the last render's +committed TOML resolves to a different branch and the pins differ, rendering +fetches that historical branch before comparing commits, without switching the +working checkout. + For static specs, rendering checks the committed `Release:` value. If it uses the `baserelease` macro, rendering copies the committed `%global baserelease` or `%define baserelease` line over the upstream definition without changing diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 5fd401240..46aac794c 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -24,6 +24,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/lockfile" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" @@ -39,10 +40,12 @@ type locklessRenderUpdate struct { } type lastRender struct { - spec string - tree *object.Tree - previous string - current string + spec string + tree *object.Tree + previous string + current string + previousBranch string + currentBranch string } type renderAutorelease interface { @@ -140,62 +143,71 @@ func readLastRender( return history, fmt.Errorf("reading last render commit:\n%w", err) } - history.previous, history.current, err = renderedUpstreamCommits(env, root, name, tree, last) + err = readRenderedUpstreamCommits(env, root, name, tree, last, &history) return history, err } -func renderedUpstreamCommits( - env *azldev.Env, root, name string, tree *object.Tree, last *object.Commit, -) (previous string, current string, err error) { +func readRenderedUpstreamCommits( + env *azldev.Env, root, name string, tree *object.Tree, last *object.Commit, history *lastRender, +) error { projectPath, err := repoRelPath(root, env.ProjectDir()) if err != nil { - return "", "", err + return err } - current, err = renderUpstreamCommit(env, tree, projectPath, name, false) + history.current, history.currentBranch, err = renderUpstreamCommit(env, tree, projectPath, name, false) if err != nil { - return "", "", fmt.Errorf("reading current render upstream commit:\n%w", err) + return fmt.Errorf("reading current render upstream commit:\n%w", err) } lastTree, err := last.Tree() if err == nil { - previous, err = renderUpstreamCommit(env, lastTree, projectPath, name, true) + history.previous, history.previousBranch, err = renderUpstreamCommit(env, lastTree, projectPath, name, true) } if err != nil { slog.Warn("Cannot read last render upstream commit; treating upstream as changed", "component", name, "error", err) - previous = "" + history.previous = "" + history.previousBranch = "" } - return previous, current, nil + return nil } func renderUpstreamCommit( env *azldev.Env, tree *object.Tree, projectPath, name string, allowLockFallback bool, -) (string, error) { +) (commit string, branch string, err error) { historicalEnv, err := loadHistoricalEnv(env, tree, projectPath) if err != nil { - return "", err + return "", "", err } comp, err := components.NewResolver(historicalEnv).GetComponentByName(name) if err != nil { - return "", fmt.Errorf("resolving historical component %#q:\n%w", name, err) + return "", "", fmt.Errorf("resolving historical component %#q:\n%w", name, err) } - commit := comp.GetConfig().Spec.UpstreamCommit + commit = comp.GetConfig().Spec.UpstreamCommit if commit == "" && allowLockFallback { - return lastRenderLockedUpstreamCommit(historicalEnv, name) + commit, err = lastRenderLockedUpstreamCommit(historicalEnv, name) + if err != nil { + return "", "", err + } } if commit == "" { - return "", fmt.Errorf("component %#q has no configured 'spec.upstream-commit'", name) + return "", "", fmt.Errorf("component %#q has no configured 'spec.upstream-commit'", name) + } + + distro, err := sourceproviders.ResolveDistro(historicalEnv, comp) + if err != nil { + return "", "", fmt.Errorf("resolving historical upstream branch:\n%w", err) } - return commit, nil + return commit, distro.Version.DistGitBranch, nil } func lastRenderLockedUpstreamCommit(env *azldev.Env, name string) (string, error) { @@ -319,6 +331,13 @@ func collectRenderUpstreamChanges( return nil } + if history.previousBranch != "" && history.previousBranch != history.currentBranch { + refspec := "+refs/heads/" + history.previousBranch + ":refs/remotes/origin/" + history.previousBranch + if _, err := git.RunInDir(env, env, dir, "fetch", "--no-tags", "--", "origin", refspec); err != nil { + return fmt.Errorf("fetching last render upstream branch %#q:\n%w", history.previousBranch, err) + } + } + changes, err := git.RunInDir(env, env, dir, "log", "--oneline", "--end-of-options", history.previous+".."+history.current, "--") if err != nil { diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 0fb506d9d..9fa0a9f02 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -37,21 +37,31 @@ func renderTestSpec(version, release, changelog string) string { func TestReadLastRender_CommittedConfigAndSpec(t *testing.T) { root := []byte(`includes = ["component.toml", "pins/*.toml"] [project] -default-distro = { name = "test", version = "1" } +default-distro = { name = "test", version = "3" } rendered-specs-dir = "SPECS" [distros.test.versions."1"] release-ver = "1" +dist-git-branch = "f40" +[distros.test.versions."2"] +release-ver = "2" +dist-git-branch = "f41" +[distros.test.versions."3"] +release-ver = "3" +dist-git-branch = "f42" `) oldSpec := renderTestSpec("1", "9%{?dist}", "* Previous\n- Keep exactly\n") repo, _ := testRepoWithCommits(t, []testRepoCommit{ {files: map[string][]byte{ - "azldev.toml": root, - "component.toml": []byte("[components.curl.spec]\ntype = 'upstream'\n"), + "azldev.toml": root, + "component.toml": []byte("[components.curl.spec]\ntype = 'upstream'\n" + + "upstream-distro = { name = 'test', version = '1' }\n"), "pins/curl.toml": []byte("[components.curl.spec]\nupstream-commit = 'aaaaaaa'\n"), "SPECS/c/curl/curl.spec": []byte(oldSpec), }}, {files: map[string][]byte{ "pins/curl.toml": []byte("[components.curl.spec]\nupstream-commit = 'bbbbbbb'\n"), + "component.toml": []byte("[components.curl.spec]\ntype = 'upstream'\n" + + "upstream-distro = { name = 'test', version = '2' }\n"), }}, {files: map[string][]byte{"README": []byte("unrelated later commit")}}, }) @@ -75,6 +85,8 @@ release-ver = "1" assert.Equal(t, oldSpec, content) assert.Equal(t, "aaaaaaa", history.previous) assert.Equal(t, "bbbbbbb", history.current) + assert.Equal(t, "f40", history.previousBranch) + assert.Equal(t, "f41", history.currentBranch) } func TestReadLastRender_HistoricalLockFallback(t *testing.T) { @@ -85,6 +97,7 @@ default-distro = { name = "test", version = "1" } lock-dir = "../historical-locks" [distros.test.versions."1"] release-ver = "1" +dist-git-branch = "f40" ` ) for _, testCase := range []struct { @@ -139,8 +152,9 @@ release-ver = "1" repo, _ := testRepoWithCommits(t, []testRepoCommit{ {files: files}, {files: map[string][]byte{ - "components.toml": []byte(current), - "settings/project.toml": []byte(strings.ReplaceAll(config, "historical-locks", "current-locks")), + "components.toml": []byte(current), + "settings/project.toml": []byte(strings.ReplaceAll( + strings.ReplaceAll(config, "historical-locks", "current-locks"), "f40", "f41")), "lockless-components.txt": []byte(selection), "historical-locks/curl.lock": []byte("version = 1\nupstream-commit = 'changed-old-lock'\n"), "current-locks/curl.lock": []byte("version = 1\nupstream-commit = 'current-lock'\n"), @@ -165,6 +179,89 @@ release-ver = "1" require.NoError(t, err) assert.Equal(t, testCase.want, history.previous) assert.Equal(t, "bbbbbbb", history.current) + assert.Equal(t, "f41", history.currentBranch) + + if testCase.want != "" { + assert.Equal(t, "f40", history.previousBranch) + } + }) + } +} + +func TestCollectRenderUpstreamChanges_Branches(t *testing.T) { + for _, testCase := range []struct { + name, previous, current, oldBranch, newBranch string + wantFetch, fetchFails, wantLog bool + }{ + { + name: "same branch", previous: "old", current: "new", + oldBranch: "f41", newBranch: "f41", wantLog: true, + }, + { + name: "branch change", previous: "old", current: "new", + oldBranch: "f40", newBranch: "f41", wantFetch: true, wantLog: true, + }, + { + name: "branch fetch fails", previous: "old", current: "new", + oldBranch: "f40", newBranch: "f41", wantFetch: true, fetchFails: true, + }, + { + name: "same commit on different branch", previous: "same", current: "same", + oldBranch: "f40", newBranch: "f41", + }, + {name: "initial render", current: "new", newBranch: "f41"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + fetchErr := errors.New("fetch failed") + + var fetched, logged bool + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[3] == "fetch" { + fetched = true + + assert.False(t, logged, "fetch must run before comparing commits") + assert.Equal(t, []string{ + "git", "-C", "/working", "fetch", "--no-tags", "--", "origin", + "+refs/heads/f40:refs/remotes/origin/f40", + }, cmd.Args) + + if testCase.fetchFails { + return "", fetchErr + } + + return "", nil + } + + logged = true + + assert.Equal(t, []string{ + "git", "-C", "/working", "log", "--oneline", "--end-of-options", "old..new", "--", + }, cmd.Args) + + return "abc123 Upstream change\n", nil + } + update := &locklessRenderUpdate{} + + err := collectRenderUpstreamChanges(env.Env, "/working", "curl", "1", + lastRender{ + previous: testCase.previous, current: testCase.current, + previousBranch: testCase.oldBranch, currentBranch: testCase.newBranch, + }, update) + if testCase.fetchFails { + require.ErrorIs(t, err, fetchErr) + require.ErrorContains(t, err, "fetching last render upstream branch") + } else { + require.NoError(t, err) + } + + assert.Equal(t, testCase.wantFetch, fetched) + assert.Equal(t, testCase.wantLog, logged) + + if testCase.wantLog { + assert.Equal(t, "abc123 Upstream change", update.changes) + } }) } } diff --git a/internal/utils/git/git.go b/internal/utils/git/git.go index 7d5c780ea..189ed3aa6 100644 --- a/internal/utils/git/git.go +++ b/internal/utils/git/git.go @@ -204,10 +204,11 @@ func resolveCloneOptions(options []GitOptions) cloneOptions { return resolved } -// WithGitBranch returns a [GitOptions] that specifies the branch to clone. +// WithGitBranch returns a [GitOptions] that clones only the specified branch +// and its full history. func WithGitBranch(branch string) GitOptions { return func(opts *cloneOptions) { - opts.args = append(opts.args, "--branch", branch) + opts.args = append(opts.args, "--branch", branch, "--single-branch") } } diff --git a/internal/utils/git/git_test.go b/internal/utils/git/git_test.go index 08ded6879..eac1d21c0 100644 --- a/internal/utils/git/git_test.go +++ b/internal/utils/git/git_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx/opctx_test" "github.com/microsoft/azure-linux-dev-tools/internal/utils/externalcmd" "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" @@ -30,7 +31,6 @@ const ( testRepoBranch = "test" testRepoSubDir = "repo" testRepoReadmeFile = "README" - testRepoContribFile = "CONTRIBUTING.md" testGitDir = ".git" nonExistentRepoURL = "https://github.com/nonexistent/repository-does-not-exist-12345.git" errMsgEmptyURL = "repository URL cannot be empty" @@ -115,19 +115,22 @@ func TestClone(t *testing.T) { } func TestCloneWithBranch(t *testing.T) { - ctrl := gomock.NewController(t) - - cmdFactory, err := externalcmd.NewCmdFactory( - opctx_test.NewNoOpMockDryRunnable(ctrl), - opctx_test.NewNoOpMockEventListener(ctrl), - ) + env := testutils.NewTestEnv(t) + provider, err := git.NewGitProviderImpl(env.EventListener, env.CmdFactory) require.NoError(t, err) - provider, err := git.NewGitProviderImpl(opctx_test.NewNoOpMockEventListener(ctrl), cmdFactory) - require.NoError(t, err) + const destDir = "/working/repo" + + called := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + called = true + + assert.Equal(t, []string{ + "git", "clone", "--branch", testRepoBranch, "--single-branch", testRepoURL, destDir, + }, cmd.Args) - tempDir := t.TempDir() - destDir := filepath.Join(tempDir, testRepoSubDir) + return nil + } err = provider.Clone( context.Background(), @@ -137,9 +140,7 @@ func TestCloneWithBranch(t *testing.T) { ) require.NoError(t, err) - assert.DirExists(t, destDir) - assert.DirExists(t, filepath.Join(destDir, testGitDir)) - assert.FileExists(t, filepath.Join(destDir, testRepoContribFile)) + assert.True(t, called) } func TestCloneNonExistentRepo(t *testing.T) { From 5d938b6bf19a6dded234ab139bbfa770f54659e3 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 21:09:53 +0000 Subject: [PATCH 12/70] perf(build): run validation targets in parallel Build once before running unit tests, default quality checks, and scenario tests concurrently through Mage's shared dependency graph. Generate docs only after all validation targets pass, preserving the existing failure gate and prerequisite ordering. Resolve tool paths with Go's -C option instead of changing the Mage process working directory, preventing concurrent targets from running in a tool module by accident. Document the pipeline and concurrency controls. Build, unit tests, and mage check all passed. Parallel execution was verified from the Mage output. Scenario tests remain blocked by the unavailable Docker provider. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/developer/reference/build.md | 9 +++++++- magefiles/magefile.go | 6 ++++-- magefiles/mageutil/mageutil.go | 36 ++----------------------------- 3 files changed, 14 insertions(+), 37 deletions(-) diff --git a/docs/developer/reference/build.md b/docs/developer/reference/build.md index 965bd46a6..8163b4122 100644 --- a/docs/developer/reference/build.md +++ b/docs/developer/reference/build.md @@ -16,7 +16,7 @@ This repo contains configs for the `golangci-lint` linter. It is installed as pa | Command | Description | When to Use | |---------|-------------|-------------| | `mage -l` | List all available targets | To see all options | -| `mage all` | Build, test, default checks, and scenario pipeline | Before submitting PR | +| `mage all` | Build, then run unit tests, default checks, and scenario tests in parallel; generate docs after they pass | Before submitting PR | | `mage build` | Build Go binaries | After code changes | | `mage install` | Build Go binaries and install | Outer-loop testing | | `mage unit` | Run unit tests | After writing tests | @@ -33,6 +33,13 @@ This repo contains configs for the `golangci-lint` linter. It is installed as pa **Note**: Dependencies run automatically in correct order (i.e. `mage build` and `mage unit` automatically run code generation as needed). +`mage all` shares one code-generation and build pass across its parallel +validation targets. Each validation target must pass before documentation is +regenerated. It still runs the default Go checks, not the optional Python checks; +use `mage check all` to include those. Parallel validation can increase peak +resource use; set `GOMAXPROCS` and `GOFLAGS=-p=` to limit Go concurrency +on memory-constrained hosts. + ## Code Generation The project uses automatic code generation including: diff --git a/magefiles/magefile.go b/magefiles/magefile.go index feddd9d98..d7a3a4b71 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -48,8 +48,10 @@ func init() { } } -// All runs the tests, builds the code, and runs the default checks. +// All builds the code, runs tests and default checks in parallel, then generates docs. func All() error { - mg.SerialDeps(magebuild.Build, magebuild.Unit, mg.F(magecheckfix.Check, magecheckfix.TargetDefault), magescenario.Scenario, magebuild.Docs) + mg.SerialDeps(magebuild.Build) + mg.Deps(magebuild.Unit, mg.F(magecheckfix.Check, magecheckfix.TargetDefault), magescenario.Scenario) + mg.SerialDeps(magebuild.Docs) return nil } diff --git a/magefiles/mageutil/mageutil.go b/magefiles/mageutil/mageutil.go index 2e1cddda4..28503f4ff 100644 --- a/magefiles/mageutil/mageutil.go +++ b/magefiles/mageutil/mageutil.go @@ -257,45 +257,13 @@ func createDir(dir string) error { // GetToolAbsPath returns the command to run a go tool that is managed within // one of the tools-focused modules in this repository. func GetToolAbsPath(toolName string) (toolPath string, err error) { - var origWd string - - // Save the current working directory so we can restore it later. - origWd, err = os.Getwd() - if err != nil { - return "", fmt.Errorf("%w: could not get current working directory: %w", ErrCmdLookup, err) - } - - // Restore the original working directory before returning. - defer func() { - chdirErr := os.Chdir(origWd) - if chdirErr != nil { - if err == nil { - err = fmt.Errorf("%w: could not restore original working directory %q: %w", - ErrCmdLookup, origWd, chdirErr) - } else { - err = fmt.Errorf("%w: could not restore original working directory %q: %w; original error: %w", - ErrCmdLookup, origWd, chdirErr, err) - } - } - }() + toolWd := filepath.Join(AzldevProjectDir(), "tools", toolName) // NOTE: We've found that we typically get a temporary-and-no-longer-resolvable path back // the first time we invoke "go tool". Thereafter, it's in the cache. We call a second // time for the *good* path. for range 2 { - var err error - - toolWd := filepath.Join(origWd, "tools", toolName) - - // NOTE: To avoid errors with "go" not being willing to handle the -modfile flag in case - // a go workspace (go.work) is present, we chdir to the directory where we expect to find - // the go.mod. - err = os.Chdir(filepath.Join(origWd, "tools", toolName)) - if err != nil { - return "", fmt.Errorf("%w: could not change directory to %q: %w", ErrCmdLookup, toolWd, err) - } - - toolPath, err = sh.Output(mg.GoCmd(), "tool", "-n", toolName) + toolPath, err = sh.Output(mg.GoCmd(), "-C", toolWd, "tool", "-n", toolName) if err != nil { return "", fmt.Errorf("%w: could not find the executable path for tool %q using cwd %q: %w", ErrCmdLookup, toolName, toolWd, err) From f7d7c8b20567ace9bb6c478156c36266f281b8ca Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 21:24:04 +0000 Subject: [PATCH 13/70] fix(component): preserve all lockless render files Retain every prepared dist-git file except .git for lockless renders, including unreferenced sidecars, hidden files, and subdirectories. Remove the host spectool query and bypass file filtering regardless of the skip-file-filter setting. Leave locked rendering unchanged. Update regression coverage, emitted agent guidance, user documentation, and generated CLI documentation and configuration schema. Validation: unit tests, mage check all, and mage docs passed. mage all remains blocked by the unavailable Docker scenario provider. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 9 ++-- .../reference/cli/azldev_component_render.md | 3 +- docs/user/reference/config/components.md | 10 ++-- .../app/azldev/agentskill/agentskill_test.go | 3 ++ .../azldev/agentskill/content/azldev.md.tmpl | 5 +- .../agentskill/content/comp-toml.md.tmpl | 8 +-- .../content/refresh-upstream-commit.md.tmpl | 3 +- .../content/withoutlockfile/azldev.md.tmpl | 3 +- .../content/withoutlockfile/comp-toml.md.tmpl | 8 +-- internal/app/azldev/cmds/component/render.go | 18 ++++--- .../app/azldev/cmds/component/render_host.go | 20 +------- .../component/render_host_internal_test.go | 51 +++---------------- .../component/render_modes_internal_test.go | 36 ++++++------- internal/projectconfig/render.go | 7 +-- schemas/azldev.schema.json | 2 +- 15 files changed, 73 insertions(+), 113 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 27e67970d..1e16221ad 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -172,10 +172,11 @@ It then always invokes host `rpmdev-bumpspec -c` with `updated ` followed by upstream changes on separate lines. The tool's no-op for `%autorelease` specs is accepted. -Existing overlays and sidecar filtering run after these steps. Synthetic history -and its legacy release adjustment are skipped. Host `spectool` supplies the -sidecar keep-list, with `changelog` always -preserved. The spec is **not** preprocessed with `rpmautospec process-distgit`: +Existing overlays run after these steps. All prepared dist-git files are retained +except `.git`, including unreferenced sidecars, hidden files, and subdirectories. +Lockless rendering does not run `spectool` or filter files; `render.skip-file-filter` +has no effect on lockless components. Synthetic history and its legacy release +adjustment are skipped. The spec is **not** preprocessed with `rpmautospec process-distgit`: `%autorelease`, `%autochangelog`, and other RPM macros remain unexpanded. No mock configuration is needed for lockless rendering. The renderer replaces the component output, excluding `.git`, diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 5bcdb88ca..f607ec430 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -36,7 +36,8 @@ rpmdev-bumpspec must be installed. Commit updated upstream-commit TOML before rendering. --check-only never replaces output or creates project commits. Lockless rendering runs entirely on the host: it never uses mock or rpmautospec process-distgit. The rendered spec retains its RPM macros, including -%autorelease and %autochangelog. Host spectool supplies the sidecar keep-list; +%autorelease and %autochangelog. All prepared dist-git files are retained except +.git; no file filtering or spectool query is performed. Host rpmautospec generate-changelog/calculate-release are used only when needed for release history or provenance. Locked components keep their existing behavior. diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 3d5f5dd23..d709bf614 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -171,11 +171,11 @@ behavior for every component. List every component for a fully lockless project. | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| -| Skip file filter | `skip-file-filter` | boolean | No | Disable post-render file filtering (defaults to `false`) | +| Skip file filter | `skip-file-filter` | boolean | No | Disable post-render file filtering for locked components (defaults to `false`; ignored for lockless components) | ### Skip File Filter -During rendering, azldev uses `spectool` to determine which files are referenced by `Source` and `Patch` tags in the spec, then removes unreferenced files from the rendered output. Some specs use dynamic macros (e.g., `%{fontpkgname1}`) that `spectool` cannot expand, causing it to report incorrect filenames. This results in referenced files being incorrectly removed. +When rendering locked components, azldev uses `spectool` to determine which files are referenced by `Source` and `Patch` tags in the spec, then removes unreferenced files from the rendered output. Some specs use dynamic macros (e.g., `%{fontpkgname1}`) that `spectool` cannot expand, causing it to report incorrect filenames. This results in referenced files being incorrectly removed. Set `skip-file-filter = true` to preserve all files from the dist-git checkout: @@ -184,7 +184,11 @@ Set `skip-file-filter = true` to preserve all files from the dist-git checkout: skip-file-filter = true ``` -> **Note:** This should only be used for specs with macros that `spectool` cannot resolve. For most components, the default filtering behavior is correct and keeps the rendered output clean. +> **Note:** This should only be used for specs with macros that `spectool` cannot resolve. For most locked components, the default filtering behavior is correct and keeps the rendered output clean. + +Lockless rendering does not query `spectool` or filter files: it always retains +all prepared dist-git files except `.git`. This setting has no effect on +components selected by `lockless-components.txt`. ## Build Configuration diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 9334ba3fb..88ec12d5f 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -177,6 +177,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "host `rpmdev-bumpspec`") assert.Contains(t, doc, "`--check-only` never creates project commits") assert.Contains(t, doc, "never uses mock or `rpmautospec process-distgit`") + assert.Contains(t, doc, "all prepared dist-git files except `.git`") + assert.NotContains(t, doc, "host `spectool` filtering") } doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) @@ -188,6 +190,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "retains `%autorelease`, `%autochangelog`") assert.Contains(t, doc, "effective upstream distro's `dist-git-branch`") assert.Contains(t, doc, "fetches that historical branch before comparing commits") + assert.Contains(t, doc, "All prepared dist-git files except `.git` are retained") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index a9d36acfd..5f7fb5870 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -40,10 +40,11 @@ generated upstream-commit TOML instead. All others use locks. For listed upstream components, rendering carries release/changelog state from the committed output, runs host `rpmdev-bumpspec`, then applies the existing -overlays and host `spectool` filtering, without synthetic history or its legacy +overlays, without synthetic history or its legacy release adjustment. Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. -It preserves the `changelog` sidecar and replaces and +It retains all prepared dist-git files except `.git`, without file filtering or +`spectool` queries, and replaces and automatically commits only that component's output directory; unrelated staged files are excluded. Commit refreshed TOML before rendering. Configure Git author identity and install `rpmdevtools`. `--check-only` never creates project commits. diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 35cf0380c..71672d04b 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -31,7 +31,7 @@ Add sub-tables only for what you change. The fields you will reach for most: | `overlays` / `overlay-files` | targeted spec/source edits (see the `azldev-overlays` skill) | | `build.defines` / `build.with` / `build.without` | RPM macro and bcond build tweaks | | `release.calculation` | how the `Release` tag is managed | -| `render.skip-file-filter` | rendering edge-case escape hatch | +| `render.skip-file-filter` | locked-rendering edge-case escape hatch | ## Spec source @@ -106,9 +106,11 @@ locks. List every component to use a fully lockless workflow. Use `comp refresh-upstream-commit` for listed components and `comp update` for unlisted components; each command skips the other mode. -`render.skip-file-filter = true` keeps all source and patch files during render. -azldev normally prunes files not referenced by the rendered spec; set this only for +`render.skip-file-filter = true` keeps all source and patch files during locked render. +Locked rendering normally prunes files not referenced by the rendered spec; set this only for the rare spec whose `Source`/`Patch` filenames use macros the filter cannot expand. +Lockless rendering always retains all prepared dist-git files except `.git`, +without file filtering or `spectool` queries; this setting has no effect on it. ## File organization diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 15f764006..bef14cf42 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -78,7 +78,8 @@ macro-expanded Release as `-b` and its committed `%changelog` body as the working `changelog` file, removing leading and trailing comment lines starting with `##` while preserving interior comments. Expansion uses the committed files on the host; the later version-change release reset still applies. Existing overlays -and host `spectool` filtering still run afterward. Synthetic history and its +still run afterward. All prepared dist-git files except `.git` are retained, +without file filtering or `spectool` queries. Synthetic history and its legacy release adjustment are skipped during lockless rendering. Lockless rendering never uses mock or `rpmautospec process-distgit`; the output retains `%autorelease`, `%autochangelog`, and other RPM macros unexpanded, and diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 28a9ad910..30298bd45 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -39,7 +39,8 @@ It replaces and automatically commits only the component output directory. Commit refreshed TOML before rendering; configure Git author identity and install `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros -remain unexpanded and the `changelog` sidecar is preserved. Synthetic history +remain unexpanded. Lockless rendering retains all prepared dist-git files except `.git`, +without file filtering or `spectool` queries. Synthetic history and its legacy release adjustment are skipped during rendering. `--check-only` never creates project commits. Local components skip upstream carryover and automatic commits. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index e8ba1be28..37087bddf 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -31,7 +31,7 @@ Add sub-tables only for what you change. The fields you will reach for most: | `overlays` / `overlay-files` | targeted spec/source edits (see the `azldev-overlays` skill) | | `build.defines` / `build.with` / `build.without` | RPM macro and bcond build tweaks | | `release.calculation` | how the `Release` tag is managed | -| `render.skip-file-filter` | rendering edge-case escape hatch | +| `render.skip-file-filter` | locked-rendering edge-case escape hatch | ## Spec source @@ -102,9 +102,11 @@ all others use lock files. A missing or empty list means all components use locks. List every component to use a fully lockless workflow. Use `comp refresh-upstream-commit` for lockless components. -`render.skip-file-filter = true` keeps all source and patch files during render. -azldev normally prunes files not referenced by the rendered spec; set this only for +`render.skip-file-filter = true` keeps all source and patch files during locked render. +Locked rendering normally prunes files not referenced by the rendered spec; set this only for the rare spec whose `Source`/`Patch` filenames use macros the filter cannot expand. +Lockless rendering always retains all prepared dist-git files except `.git`, +without file filtering or `spectool` queries; this setting has no effect on it. ## File organization diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index a34b1b98c..d6697f9d6 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -85,7 +85,8 @@ rpmdev-bumpspec must be installed. Commit updated upstream-commit TOML before rendering. --check-only never replaces output or creates project commits. Lockless rendering runs entirely on the host: it never uses mock or rpmautospec process-distgit. The rendered spec retains its RPM macros, including -%autorelease and %autochangelog. Host spectool supplies the sidecar keep-list; +%autorelease and %autochangelog. All prepared dist-git files are retained except +.git; no file filtering or spectool query is performed. Host rpmautospec generate-changelog/calculate-release are used only when needed for release history or provenance. Locked components keep their existing behavior. @@ -176,7 +177,7 @@ const ( // RenderComponents renders the post-overlay spec and sidecar files for each // selected component into the output directory. Processing is done in four phases: // 1. Parallel source preparation (clone, overlay, synthetic git) -// 2. Host sidecar queries for lockless components; batch mock processing for locked components +// 2. Batch mock processing for locked components only // 3. Parallel finishing (filter files, remove .git, copy output) // 4. Serial project commits for lockless upstream component output func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, error) { @@ -792,7 +793,7 @@ func finishComponentRender( componentDir := filepath.Join(stagingDir, componentName) specPath := filepath.Join(componentDir, prep.specFilename) - // Check the host or mock source-query result. + // Check the source-processing result. mockResult, hasMockResult := mockResultMap[componentName] if !hasMockResult { return false, fmt.Errorf( @@ -804,12 +805,15 @@ func finishComponentRender( "source processing failed for %#q:\n%w", componentName, mockResult.Error) } - // Filter files using the host or mock spectool result. + // Filter locked components using the mock spectool result. // Skip filtering when: - // 1. The component config explicitly opts out via 'skip-file-filter'. - // 2. spectool output contains unexpanded RPM macros (%{...}), indicating + // 1. The component is lockless. + // 2. The component config explicitly opts out via 'skip-file-filter'. + // 3. spectool output contains unexpanded RPM macros (%{...}), indicating // that the reported filenames don't match the real files on disk. - if prep.comp.GetConfig().Render.SkipFileFilter { + if prep.lockless { + slog.Debug("Preserving all files for lockless render", "component", componentName) + } else if prep.comp.GetConfig().Render.SkipFileFilter { slog.Info("Skipping file filter ('skip-file-filter' is set)", "component", componentName) } else if macro := findUnexpandedMacro(mockResult.SpecFiles); macro != "" { slog.Info("Skipping file filter (spectool output contains unexpanded macros)", diff --git a/internal/app/azldev/cmds/component/render_host.go b/internal/app/azldev/cmds/component/render_host.go index 4f47c32c8..9c22ea34f 100644 --- a/internal/app/azldev/cmds/component/render_host.go +++ b/internal/app/azldev/cmds/component/render_host.go @@ -8,12 +8,10 @@ import ( "context" "fmt" "os/exec" - "path/filepath" "strings" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" - "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spectool" "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" ) @@ -50,15 +48,6 @@ func (p hostRenderProcessor) ExpandRelease(ctx context.Context, dir, filename st return "", fmt.Errorf("no Release field in committed spec query output %#q", out) } -func (p hostRenderProcessor) specFiles(ctx context.Context, dir, filename string) ([]string, error) { - out, err := p.run(ctx, dir, "spectool", "--define", "_sourcedir "+dir, "-l", "-a", filename) - if err != nil { - return nil, err - } - - return append(spectool.ParseSpectoolOutput(out), "changelog"), nil -} - func (p hostRenderProcessor) run(ctx context.Context, dir, binary string, args ...string) (string, error) { if err := prereqs.RequireExecutable(p.env, binary, nil); err != nil { return "", fmt.Errorf("lockless rendering requires host program %#q:\n%w", binary, err) @@ -90,8 +79,6 @@ func processRenderSources( var locked []*preparedComponent results := make(map[string]*sources.ComponentMockResult, len(prepared)) - host := hostRenderProcessor{env: env} - for _, prep := range prepared { if !prep.lockless { locked = append(locked, prep) @@ -101,12 +88,7 @@ func processRenderSources( name := prep.comp.GetName() - result := &sources.ComponentMockResult{Name: name} - if !prep.comp.GetConfig().Render.SkipFileFilter { - result.SpecFiles, result.Error = host.specFiles(env, filepath.Join(stagingDir, name), prep.specFilename) - } - - results[name] = result + results[name] = &sources.ComponentMockResult{Name: name} } for name, result := range batchMockProcess(env, processor, stagingDir, locked) { diff --git a/internal/app/azldev/cmds/component/render_host_internal_test.go b/internal/app/azldev/cmds/component/render_host_internal_test.go index 819469312..983913887 100644 --- a/internal/app/azldev/cmds/component/render_host_internal_test.go +++ b/internal/app/azldev/cmds/component/render_host_internal_test.go @@ -12,7 +12,6 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" - "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/stretchr/testify/assert" @@ -111,33 +110,12 @@ func TestHostRenderExpandRelease(t *testing.T) { } } -func TestHostRenderSpecFiles(t *testing.T) { - env := testutils.NewTestEnv(t) - env.CmdFactory.RegisterCommandInSearchPath("spectool") - env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { - assert.Equal(t, "/working", cmd.Dir) - assert.Equal(t, []string{ - "spectool", "--define", "_sourcedir /working", "-l", "-a", "curl.spec", - }, cmd.Args) - - return "Source0: https://example.com/curl-1.tar.gz\nPatch0: fixes/fix.patch\n", nil - } - files, err := (hostRenderProcessor{env: env.Env}).specFiles(env.Env, "/working", "curl.spec") - require.NoError(t, err) - assert.Equal(t, []string{"curl-1.tar.gz", "fixes/fix.patch", "changelog"}, files) -} - func TestMixedRenderRoutesOnlyLockedComponentsThroughMock(t *testing.T) { - const ( - hostFailure = "host failure" - mockFailure = "mock failure" - skipFilter = "skip host filter" - ) + const mockFailure = "mock failure" - for _, phase := range []string{"success", hostFailure, mockFailure, skipFilter} { + for _, phase := range []string{"success", mockFailure} { t.Run(phase, func(t *testing.T) { env := testutils.NewTestEnv(t) - env.CmdFactory.RegisterCommandInSearchPath("spectool") env.CmdFactory.RegisterCommandInSearchPath("mock") ctrl := gomock.NewController(t) @@ -146,26 +124,16 @@ func TestMixedRenderRoutesOnlyLockedComponentsThroughMock(t *testing.T) { host := components_testutils.NewMockComponent(ctrl) host.EXPECT().GetName().AnyTimes().Return("rust-asyncband") - host.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ - Render: projectconfig.ComponentRenderConfig{SkipFileFilter: phase == skipFilter}, - }) env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} processor, err := renderMockProcessor(env.Env, []components.Component{host, locked}) require.NoError(t, err) require.NotNil(t, processor) - queryErr := errors.New("host query failed") env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { - assert.NotEqual(t, skipFilter, phase) - assert.Equal(t, "spectool", cmd.Args[0]) - assert.Equal(t, "/stage/rust-asyncband", cmd.Dir) + t.Fatalf("unexpected host query during lockless rendering: %v", cmd.Args) - if phase == hostFailure { - return "", queryErr - } - - return "Patch0: host.patch\n", nil + return "", nil } batched := false env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { @@ -195,15 +163,8 @@ func TestMixedRenderRoutesOnlyLockedComponentsThroughMock(t *testing.T) { require.Contains(t, results, "rust-asyncband") assert.Equal(t, phase != mockFailure, batched) - if phase == hostFailure { - require.ErrorIs(t, results["rust-asyncband"].Error, queryErr) - } else { - require.NoError(t, results["rust-asyncband"].Error) - - if phase != skipFilter { - assert.Equal(t, []string{"host.patch", "changelog"}, results["rust-asyncband"].SpecFiles) - } - } + require.NoError(t, results["rust-asyncband"].Error) + assert.Empty(t, results["rust-asyncband"].SpecFiles) if phase == mockFailure { assert.NotContains(t, results, "curl") diff --git a/internal/app/azldev/cmds/component/render_modes_internal_test.go b/internal/app/azldev/cmds/component/render_modes_internal_test.go index 31147656e..36dfef068 100644 --- a/internal/app/azldev/cmds/component/render_modes_internal_test.go +++ b/internal/app/azldev/cmds/component/render_modes_internal_test.go @@ -88,7 +88,7 @@ func (p *renderModeObservingPreparer) PrepareSources( return p.err } -func TestLocklessRenderPreservesMacrosWithoutMock(t *testing.T) { +func TestLocklessRenderPreservesAllFilesWithoutMock(t *testing.T) { env := testutils.NewTestEnvWithoutLockfile(t) version := env.Config.Distros["test-distro"].Versions["1.0"] version.MockConfigPath = "" @@ -97,15 +97,10 @@ func TestLocklessRenderPreservesMacrosWithoutMock(t *testing.T) { Name: "curl", Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, } - env.CmdFactory.RegisterCommandInSearchPath("spectool") - - queries := 0 env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { - queries++ - - assert.Equal(t, "spectool", cmd.Args[0]) + t.Fatalf("unexpected query during lockless rendering: %v", cmd.Args) - return "Patch0: fix.patch\n", nil + return "", nil } env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { t.Fatalf("unexpected command during host-only rendering: %v", cmd.Args) @@ -118,6 +113,12 @@ func TestLocklessRenderPreservesMacrosWithoutMock(t *testing.T) { changelog = "* Old entry\n- Keep unchanged\n" ) + files := map[string]string{ + "curl.spec": rawSpec, "changelog": changelog, "fix.patch": "patch", + "unreferenced": "keep", ".gitignore": "upstream ignore rules", + "extras/nested.txt": "unreferenced nested file", + } + original := newSourcePreparer newSourcePreparer = func( _ sourceproviders.SourceManager, fs opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, @@ -126,15 +127,13 @@ func TestLocklessRenderPreservesMacrosWithoutMock(t *testing.T) { return &renderModeObservingPreparer{ observe: func(_ components.Component) {}, prepare: func(_ components.Component, dir string) error { - for name, content := range map[string]string{ - "curl.spec": rawSpec, "changelog": changelog, "fix.patch": "patch", - "unreferenced": "discard", ".git/HEAD": "temporary history", - } { + for name, content := range files { require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, name), []byte(content), fileperms.PublicFile)) } - return nil + return fileutils.WriteFile(fs, filepath.Join(dir, ".git/HEAD"), + []byte("temporary history"), fileperms.PublicFile) }, }, nil } @@ -148,19 +147,16 @@ func TestLocklessRenderPreservesMacrosWithoutMock(t *testing.T) { require.NoError(t, err) require.Len(t, results, 1) assert.Equal(t, renderStatusOK, results[0].Status) - assert.Equal(t, 1, queries) - for name, want := range map[string]string{"curl.spec": rawSpec, "changelog": changelog, "fix.patch": "patch"} { + for name, want := range files { content, err := fileutils.ReadFile(env.TestFS, filepath.Join("/rendered/c/curl", name)) require.NoError(t, err) assert.Equal(t, want, string(content)) } - for _, name := range []string{".git", "unreferenced"} { - exists, err := fileutils.Exists(env.TestFS, filepath.Join("/rendered/c/curl", name)) - require.NoError(t, err) - assert.False(t, exists) - } + exists, err := fileutils.Exists(env.TestFS, "/rendered/c/curl/.git") + require.NoError(t, err) + assert.False(t, exists) } func TestRenderComponentsSelectsLockModeFromFile(t *testing.T) { diff --git a/internal/projectconfig/render.go b/internal/projectconfig/render.go index c623f8afc..0f377e6cd 100644 --- a/internal/projectconfig/render.go +++ b/internal/projectconfig/render.go @@ -6,10 +6,11 @@ package projectconfig // ComponentRenderConfig encapsulates configuration for rendering a component. type ComponentRenderConfig struct { // SkipFileFilter, when true, disables the post-render file filter for this - // component. Normally, rendered output is filtered to only include files + // locked component. Normally, locked rendered output is filtered to only include files // referenced by Source/Patch tags in the spec (as reported by spectool). // Some specs use macros that spectool cannot expand, causing referenced // files to be incorrectly removed. Setting this to true preserves all - // files from the dist-git checkout. - SkipFileFilter bool `toml:"skip-file-filter,omitempty" json:"skipFileFilter,omitempty" jsonschema:"title=Skip file filter,description=Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags"` + // files from the dist-git checkout. Lockless rendering always preserves all + // prepared files except '.git', regardless of this setting. + SkipFileFilter bool `toml:"skip-file-filter,omitempty" json:"skipFileFilter,omitempty" jsonschema:"title=Skip file filter,description=Disable post-render file filtering for locked components with unexpandable macros in Source/Patch tags. Lockless rendering always preserves all prepared files except .git."` } diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 9cf47bd35..795c1ea39 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -345,7 +345,7 @@ "skip-file-filter": { "type": "boolean", "title": "Skip file filter", - "description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags" + "description": "Disable post-render file filtering for locked components with unexpandable macros in Source/Patch tags. Lockless rendering always preserves all prepared files except .git." } }, "additionalProperties": false, From 0739baed303290c7841e82862f674da4377735ae Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 21:43:07 +0000 Subject: [PATCH 14/70] fix(component): capture pristine upstream provenance Resolve needed Fedora provenance immediately after fetching the pinned upstream checkout, before release/changelog carryover or bumping changes the spec. Reuse the captured macros during overlay processing instead of querying the modified checkout. Skip autorelease resolution when an explicit release macro overrides the result. Do not query provenance when overlays or provenance are disabled, and do not retry failed resolution when writing the macros file. Add in-memory regressions for pristine inputs, conditional resolution, and reuse. Update user documentation, emitted guidance, and CLI help. Validation: build and unit tests passed through mage all. mage check all and mage docs passed. Docker scenarios remain environment-blocked. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 14 +- .../reference/cli/azldev_component_render.md | 3 +- docs/user/reference/config/components.md | 7 +- .../app/azldev/agentskill/agentskill_test.go | 2 + .../content/refresh-upstream-commit.md.tmpl | 7 + internal/app/azldev/cmds/component/render.go | 3 +- .../app/azldev/core/sources/sourceprep.go | 37 ++++-- .../core/sources/upstream_provenance.go | 11 ++ .../upstream_provenance_internal_test.go | 120 ++++++++++++++++++ 9 files changed, 184 insertions(+), 20 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 1e16221ad..5a77cf2fd 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -118,8 +118,18 @@ The renderer clones only the branch selected by the component's effective `spec.upstream-distro` (or the project default) into temporary storage. The resolved distro version's `dist-git-branch` supplies the branch name. The clone retains that branch's full history, not every upstream branch, so pinned commits -and snapshot selection remain available without a shallow-history limit. It -finds the existing committed output under +and snapshot selection remain available without a shallow-history limit. + +Immediately after checking out the pinned upstream commit, rendering captures +optional Fedora provenance values for `build.emit-upstream-provenance`, before +release/changelog carryover or bumping changes the spec. If the upstream release +uses `%autorelease` and `build.defines` does not already provide +`fedora_upstream_release`, it runs host `rpmautospec calculate-release` at this +point. The captured values are reused later when writing the macros file, without +repeating the query against the modified checkout. No provenance query runs when +provenance is disabled or the component is not sourced from Fedora upstream. + +The renderer then finds the existing committed output under `//`, ignoring uncommitted spec changes. For an existing non-`%autorelease` upstream spec, it checks the committed `Release:` value. If it references the `baserelease` macro, diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index f607ec430..3ed1ff251 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -39,7 +39,8 @@ rpmautospec process-distgit. The rendered spec retains its RPM macros, including %autorelease and %autochangelog. All prepared dist-git files are retained except .git; no file filtering or spectool query is performed. Host rpmautospec generate-changelog/calculate-release are used only when needed for -release history or provenance. Locked components keep their existing behavior. +release history or provenance. Provenance is captured before carryover/bumping +and reused for the macros file. Locked components keep their existing behavior. When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index d709bf614..aa3b28ecc 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -253,9 +253,12 @@ emit-upstream-provenance = true | `%fedora_upstream_version` | The `Version` tag from the pristine upstream Fedora spec | | `%fedora_upstream_release` | The `Release` tag from the pristine upstream Fedora spec, with `%{?dist}` expanded to the Fedora dist tag (e.g. `.fc43`) | -The values are read from the upstream spec **before** any azldev overlays are applied, so they reflect the true upstream Name-Version-Release, not the Azure Linux–modified spec. The macros are derived fresh at render/build time from the pinned upstream commit and emitted into the component's generated macros file (loaded via `%{load:...}`). +The values are captured immediately after fetching the pinned upstream checkout, +**before** release/changelog carryover, release bumping, or overlays can modify +the spec. The captured values are reused when writing the component's generated +macros file (loaded via `%{load:...}`), rather than querying the modified checkout. -For specs whose `Release` uses rpmautospec (`Release: %autorelease`), the pristine Fedora release number is computed by running `rpmautospec calculate-release` against the upstream dist-git checkout (whose history is Fedora's, not azldev's synthetic overlay history). Lockless rendering runs this query on the host; other workflows use the project distro's mock chroot. If resolution is unavailable or fails, `%fedora_upstream_release` is skipped (with a warning) rather than emitting the literal `%autorelease`. +For specs whose `Release` uses rpmautospec (`Release: %autorelease`), the pristine Fedora release number is computed by running `rpmautospec calculate-release` against the upstream dist-git checkout (whose history is Fedora's, not azldev's synthetic overlay history). Lockless rendering runs this query on the host; other workflows use the project distro's mock chroot. The query runs only when overlays will be applied, provenance is enabled for a Fedora upstream component, and `build.defines` does not already supply `fedora_upstream_release`. Static releases do not need this query. If resolution is unavailable or fails, `%fedora_upstream_release` is skipped (with a warning) rather than emitting the literal `%autorelease`; the query is not retried when writing the macros file. Example: for a component pinned to Fedora 43's `grub2-2.12-5.fc43`, the spec can reference: diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 88ec12d5f..70d623d5c 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -191,6 +191,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "effective upstream distro's `dist-git-branch`") assert.Contains(t, doc, "fetches that historical branch before comparing commits") assert.Contains(t, doc, "All prepared dist-git files except `.git` are retained") + assert.Contains(t, doc, "immediately after checking out the pinned upstream commit") + assert.Contains(t, doc, "that step never repeats the provenance query") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index bef14cf42..6fcb22d86 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -61,6 +61,13 @@ committed TOML resolves to a different branch and the pins differ, rendering fetches that historical branch before comparing commits, without switching the working checkout. +With `build.emit-upstream-provenance`, rendering captures Fedora provenance +immediately after checking out the pinned upstream commit, before +release/changelog carryover or bumping. Host `rpmautospec calculate-release` +runs here only for an upstream `%autorelease` whose `fedora_upstream_release` +is not already supplied by `build.defines`. The captured values are reused +when writing the macros file; that step never repeats the provenance query. + For static specs, rendering checks the committed `Release:` value. If it uses the `baserelease` macro, rendering copies the committed `%global baserelease` or `%define baserelease` line over the upstream definition without changing diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index d6697f9d6..1a276ebf4 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -88,7 +88,8 @@ rpmautospec process-distgit. The rendered spec retains its RPM macros, including %autorelease and %autochangelog. All prepared dist-git files are retained except .git; no file filtering or spectool query is performed. Host rpmautospec generate-changelog/calculate-release are used only when needed for -release history or provenance. Locked components keep their existing behavior. +release history or provenance. Provenance is captured before carryover/bumping +and reused for the macros file. Locked components keep their existing behavior. When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 1ed6af893..4c789bf04 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -347,6 +347,9 @@ func (p *sourcePreparerImpl) PrepareSources( component.GetName(), err) } + // Capture provenance before fetched-source preparation changes the upstream spec. + macros := p.resolveMacros(ctx, component, outputDir, applyOverlays) + if err := p.prepareFetched(ctx, component, outputDir); err != nil { return fmt.Errorf("preparing fetched sources for %#q:\n%w", component.GetName(), err) } @@ -362,7 +365,7 @@ func (p *sourcePreparerImpl) PrepareSources( fingerprintConfig := component.GetConfig() if applyOverlays { - repackedArchives, err := p.applyOverlaysToSources(ctx, component, outputDir) + repackedArchives, err := p.applyOverlaysToSources(component, outputDir, macros) if err != nil { return err } @@ -413,11 +416,11 @@ func (p *sourcePreparerImpl) validateArchiveOverlayConfig(component components.C // (empty in dry-run mode or when no archive overlays ran), so the caller can // rehash exactly those entries in the 'sources' file. func (p *sourcePreparerImpl) applyOverlaysToSources( - ctx context.Context, component components.Component, outputDir string, + component components.Component, outputDir string, macros map[string]string, ) ([]string, error) { var macrosFileName string - macrosFilePath, err := p.writeMacrosFile(ctx, component, outputDir) + macrosFilePath, err := p.writeMacrosFile(component, outputDir, macros) if err != nil { return nil, fmt.Errorf("failed to write macros file for component %#q:\n%w", component.GetName(), err) @@ -818,7 +821,8 @@ func (p *sourcePreparerImpl) DiffSources( // Apply overlays in-place to the copied directory only. The repacked-archive // list is unused here: DiffSources diffs the trees directly and does not // rewrite a 'sources' file. - if _, err := p.applyOverlaysToSources(ctx, component, overlaidDir); err != nil { + macros := p.resolveMacros(ctx, component, overlaidDir, true) + if _, err := p.applyOverlaysToSources(component, overlaidDir, macros); err != nil { return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } @@ -1294,19 +1298,24 @@ func (p *sourcePreparerImpl) resolveSourceHash( return hash, hashType, nil } -// writeMacrosFile writes a macros file containing the resolved macros for a component. -// This includes with/without flags converted to macro format, and any explicit defines. -// If the build configuration produces no macros, no file is written and an empty path is -// returned. Otherwise, the path to the written macros file is returned. -func (p *sourcePreparerImpl) writeMacrosFile( - ctx context.Context, component components.Component, outputDir string, -) (string, error) { - macros := buildMacrosMap(component.GetConfig().Build) +func (p *sourcePreparerImpl) resolveMacros( + ctx context.Context, component components.Component, outputDir string, applyOverlays bool, +) map[string]string { + if !applyOverlays { + return nil + } - // Layer Fedora upstream provenance macros on top, reading the pristine - // upstream spec that is already on disk (before overlays are applied). + macros := buildMacrosMap(component.GetConfig().Build) p.addUpstreamProvenanceMacros(ctx, macros, component, outputDir) + return macros +} + +// writeMacrosFile writes the previously resolved macros, returning the file's path +// or an empty string when no macros are needed. +func (p *sourcePreparerImpl) writeMacrosFile( + component components.Component, outputDir string, macros map[string]string, +) (string, error) { contents := renderMacrosFile(macros) if contents == "" { return "", nil diff --git a/internal/app/azldev/core/sources/upstream_provenance.go b/internal/app/azldev/core/sources/upstream_provenance.go index 260293582..d775f8f9a 100644 --- a/internal/app/azldev/core/sources/upstream_provenance.go +++ b/internal/app/azldev/core/sources/upstream_provenance.go @@ -107,6 +107,13 @@ func (p *sourcePreparerImpl) addUpstreamProvenanceMacros( return } + _, versionDefined := macros[fedoraUpstreamVersionMacro] + + _, releaseDefined := macros[fedoraUpstreamReleaseMacro] + if versionDefined && releaseDefined { + return + } + specPath, err := findSpecInDir(p.fs, component, sourcesDir) if err != nil { // The user opted in via emit-upstream-provenance, so surface the @@ -130,6 +137,10 @@ func (p *sourcePreparerImpl) addUpstreamProvenanceMacros( setMacroIfAbsent(macros, fedoraUpstreamVersionMacro, version) } + if releaseDefined { + return + } + resolvedRelease := p.resolveUpstreamRelease(ctx, component.GetName(), release, sourcesDir, specPath) if resolvedRelease != "" { setMacroIfAbsent(macros, fedoraUpstreamReleaseMacro, resolvedRelease) diff --git a/internal/app/azldev/core/sources/upstream_provenance_internal_test.go b/internal/app/azldev/core/sources/upstream_provenance_internal_test.go index c0c60eae2..6f82405ac 100644 --- a/internal/app/azldev/core/sources/upstream_provenance_internal_test.go +++ b/internal/app/azldev/core/sources/upstream_provenance_internal_test.go @@ -9,7 +9,11 @@ import ( "path/filepath" "testing" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" @@ -26,6 +30,8 @@ type fakeAutoreleaseResolver struct { err error lastHostDir string lastSpecFile string + calls int + observe func(string, string) } func (f *fakeAutoreleaseResolver) CalculateRelease( @@ -34,6 +40,11 @@ func (f *fakeAutoreleaseResolver) CalculateRelease( f.lastHostDir = specHostDir f.lastSpecFile = specFilename + f.calls++ + if f.observe != nil { + f.observe(specHostDir, specFilename) + } + return f.output, f.err } @@ -354,6 +365,115 @@ func TestAddUpstreamProvenanceMacros_PlainDistTag(t *testing.T) { "unconditional %{dist} is expanded to the Fedora dist tag") } +func TestPrepareSources_CapturesPristineProvenance(t *testing.T) { + for _, testCase := range []struct { + name string + release string + override string + emit bool + applyOverlays bool + resolverErr error + wantRelease string + wantCalls int + }{ + { + name: "autorelease", release: "%autorelease -b 2", emit: true, applyOverlays: true, + wantRelease: "3.fc43", wantCalls: 1, + }, + { + name: "static release", release: "5%{?dist}", emit: true, applyOverlays: true, + wantRelease: "5.fc43", + }, + { + name: "explicit release override", release: "%autorelease", override: "42.fc43", + emit: true, applyOverlays: true, wantRelease: "42.fc43", + }, + { + name: "provenance disabled", release: "%autorelease", applyOverlays: true, + }, + { + name: "overlays disabled", release: "%autorelease", emit: true, + }, + { + name: "failed resolution is not retried", release: "%autorelease", emit: true, applyOverlays: true, + resolverErr: errors.New("release unavailable"), wantCalls: 1, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := testctx.NewCtx() + + config := upstreamCfg(testCase.emit) + if testCase.override != "" { + config.Build.Defines = map[string]string{fedoraUpstreamReleaseMacro: testCase.override} + } + + comp := mockComponent(ctrl, "fwupd-efi", config) + manager := sourceproviders_test.NewMockSourceManager(ctrl) + manager.EXPECT().FetchComponent(gomock.Any(), comp, provenanceWorkDir, gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + writeProvenanceSpec(t, ctx.FS(), "fwupd-efi", "1.6", testCase.release) + + return fileutils.WriteFile(ctx.FS(), filepath.Join(dir, ".git/HEAD"), + []byte("pinned upstream commit\n"), fileperms.PublicFile) + }) + + prepared := false + resolver := &fakeAutoreleaseResolver{ + output: "Calculated release number: 3\n", + err: testCase.resolverErr, + observe: func(dir, filename string) { + assert.False(t, prepared, "resolve provenance before release preparation") + + version, release, err := parseSpecVersionRelease(ctx.FS(), filepath.Join(dir, filename)) + require.NoError(t, err) + assert.Equal(t, "1.6", version) + assert.Equal(t, testCase.release, release) + + head, err := fileutils.ReadFile(ctx.FS(), filepath.Join(dir, ".git/HEAD")) + require.NoError(t, err) + assert.Equal(t, "pinned upstream commit\n", string(head)) + }, + } + preparer, err := NewPreparer(manager, ctx.FS(), ctx, ctx, + WithGitRepo(nil, nil, "43"), WithoutSyntheticHistory(), WithSkipLookaside(), + WithUpstreamProvenance(".fc43"), WithAutoreleaseResolver(resolver), + WithFetchedSources(func(_ context.Context, _ components.Component, _ string) error { + assert.Equal(t, testCase.wantCalls, resolver.calls) + + prepared = true + + writeProvenanceSpec(t, ctx.FS(), "fwupd-efi", "99", "99%{?dist}") + + return nil + })) + require.NoError(t, err) + require.NoError(t, preparer.PrepareSources(ctx, comp, provenanceWorkDir, testCase.applyOverlays)) + assert.True(t, prepared) + assert.Equal(t, testCase.wantCalls, resolver.calls) + + path := filepath.Join(provenanceWorkDir, "fwupd-efi"+MacrosFileExtension) + if !testCase.emit || !testCase.applyOverlays { + exists, err := fileutils.Exists(ctx.FS(), path) + require.NoError(t, err) + assert.False(t, exists) + + return + } + + contents, err := fileutils.ReadFile(ctx.FS(), path) + require.NoError(t, err) + assert.Contains(t, string(contents), "%fedora_upstream_version 1.6\n") + + if testCase.wantRelease == "" { + assert.NotContains(t, string(contents), "%fedora_upstream_release ") + } else { + assert.Contains(t, string(contents), "%fedora_upstream_release "+testCase.wantRelease+"\n") + } + }) + } +} + // upstreamCfg returns a Fedora-upstream component config with provenance // emission toggled by emit. func upstreamCfg(emit bool) *projectconfig.ComponentConfig { From 969bdd129eb4bcd1c87f2006005a83623e5bad5c Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 21:59:43 +0000 Subject: [PATCH 15/70] fix(component): replace existing rendered aliases Remove any existing encoded-name alias path before creating the relative symlink, including regular files and nonempty directories. Do not follow existing symlinks or remove their targets. Cover replacement of files, populated directories, and stale symlinks, and document the destructive replacement behavior. Validation: mage fix all and mage unit passed. Build and default checks passed through mage all; Docker scenarios remain environment-blocked. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 7 +++ internal/app/azldev/cmds/component/render.go | 39 ++------------ .../cmds/component/render_internal_test.go | 54 ++++++++++++------- 3 files changed, 46 insertions(+), 54 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 5a77cf2fd..0d8c50477 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -195,6 +195,13 @@ body lists upstream changes, if any. Unrelated staged files are not committed. Unchanged Git content does not produce an empty commit. A commit failure leaves the rendered files available for inspection. +For component names requiring URL encoding, rendering also creates a sibling +alias such as `libxml%2B%2B -> libxml++` for downstream tooling. Any existing +entry at the encoded alias path is removed first, including regular files and +nonempty directories, and replaced with a relative symlink. Existing symlink +targets are not removed. Alias creation failures produce a warning. +`--check-only` does not replace aliases. + Install `rpmdevtools`, `rpm-build`, and `rpmautospec` on the host as needed for the release/changelog queries, and configure Git author identity before rendering. Output (including a custom `--output-dir`) must be inside the project diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 1a276ebf4..8524bdd74 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "log/slog" - "os" "path/filepath" "slices" "sort" @@ -877,11 +876,8 @@ func finishComponentRender( // No-ops when no encoding is needed (plain ASCII names) or when the underlying // filesystem doesn't support symlinks (e.g., in-memory test FS). // -// Refuses to overwrite a non-symlink at the alias path — if a real component -// directory already lives there (the hypothetical 'gtk%2B' next to 'gtk+' -// case), bail with an error rather than silently destroying that component's -// rendered output. RPM names don't use '%' in practice, so this is belt-and -// suspenders. +// Removes any existing entry at the alias path, including nonempty directories, +// before creating the symlink. func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName string) error { aliasName := components.RenderedSpecDirAliasName(componentName) if aliasName == "" { @@ -899,23 +895,8 @@ func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName st parentDir := filepath.Dir(componentOutputDir) aliasPath := filepath.Join(parentDir, aliasName) - // Inspect any existing entry at the alias path. We only ever clobber a - // pre-existing symlink (a stale alias from a previous render); a real - // directory or file there means a name collision with another component - // and must be reported, not silently destroyed. - info, lstatErr := lstatIfPossible(fileSystem, aliasPath) - switch { - case lstatErr == nil && info.Mode()&os.ModeSymlink == 0: - return fmt.Errorf( - "alias path %#q is already occupied by a non-symlink entry; refusing to overwrite", - aliasPath) - case lstatErr == nil: - // Existing symlink — remove and replace below. - if removeErr := fileSystem.Remove(aliasPath); removeErr != nil { - return fmt.Errorf("removing existing alias symlink %#q:\n%w", aliasPath, removeErr) - } - case !errors.Is(lstatErr, os.ErrNotExist): - return fmt.Errorf("inspecting alias path %#q:\n%w", aliasPath, lstatErr) + if removeErr := fileSystem.RemoveAll(aliasPath); removeErr != nil { + return fmt.Errorf("removing existing alias path %#q:\n%w", aliasPath, removeErr) } // Use a relative target so the rendered tree stays portable. @@ -927,18 +908,6 @@ func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName st return nil } -// lstatIfPossible returns the link info at path without following symlinks, if -// the underlying filesystem supports it. Falls back to a regular Stat otherwise. -func lstatIfPossible(fileSystem opctx.FS, path string) (os.FileInfo, error) { - if lstater, ok := fileSystem.(afero.Lstater); ok { - info, _, err := lstater.LstatIfPossible(path) - - return info, err //nolint:wrapcheck // pass-through to the caller. - } - - return fileSystem.Stat(path) //nolint:wrapcheck // pass-through to the caller. -} - // copyRenderedOutput copies the rendered files from tempDir to the component's output directory. // For managed output (inside project root), existing output is removed before copying. // For external output, existing directories cause an error. diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 396038acc..1a734eff8 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -379,28 +379,44 @@ func TestWriteAliasSymlink(t *testing.T) { assert.Equal(t, "gtk+", osReadlink(t, osFS, aliasPath), "stale symlink should be replaced") }) - t.Run("refuses to overwrite a non-symlink at alias path", func(t *testing.T) { - osFS := afero.NewOsFs() - root := t.TempDir() - letterDir := filepath.Join(root, "g") - realDir := filepath.Join(letterDir, "gtk+") - require.NoError(t, fileutils.MkdirAll(osFS, realDir)) + const symlinkType = "symlink" + + for _, entryType := range []string{"file", "directory", symlinkType} { + t.Run("replaces existing "+entryType+" at alias path", func(t *testing.T) { + osFS := afero.NewOsFs() + letterDir := filepath.Join(t.TempDir(), "g") + realDir := filepath.Join(letterDir, "gtk+") + require.NoError(t, fileutils.MkdirAll(osFS, realDir)) + + aliasPath := filepath.Join(letterDir, "gtk%2B") + stalePath := aliasPath + + switch entryType { + case "directory": + stalePath = filepath.Join(aliasPath, "nested", "stale.txt") + case symlinkType: + stalePath = filepath.Join(letterDir, "old-target", "keep.txt") + } - // Hypothetical collision: a real component named 'gtk%2B' already exists - // at the alias path. RPM names don't use '%' in practice, but the guard - // must protect against it. - collisionDir := filepath.Join(letterDir, "gtk%2B") - require.NoError(t, fileutils.MkdirAll(osFS, collisionDir)) + require.NoError(t, fileutils.MkdirAll(osFS, filepath.Dir(stalePath))) + require.NoError(t, fileutils.WriteFile(osFS, stalePath, []byte("old contents"), fileperms.PublicFile)) - err := writeAliasSymlink(osFS, realDir, "gtk+") - require.Error(t, err) - assert.Contains(t, err.Error(), "non-symlink") + if entryType == symlinkType { + osSymlink(t, osFS, "old-target", aliasPath) + } - // Collision dir must still exist -- the guard MUST NOT have destroyed it. - exists, existsErr := fileutils.DirExists(osFS, collisionDir) - require.NoError(t, existsErr) - assert.True(t, exists, "non-symlink at alias path must be preserved") - }) + require.NoError(t, writeAliasSymlink(osFS, realDir, "gtk+")) + linkInfo := osLstat(t, osFS, aliasPath) + assert.NotZero(t, linkInfo.Mode()&os.ModeSymlink) + assert.Equal(t, "gtk+", osReadlink(t, osFS, aliasPath)) + + if entryType == symlinkType { + contents, err := fileutils.ReadFile(osFS, stalePath) + require.NoError(t, err) + assert.Equal(t, "old contents", string(contents), "removing an alias must not remove its target") + } + }) + } t.Run("no-op on filesystem without symlink support", func(t *testing.T) { // MemMapFs does NOT implement afero.Linker -- function should silently no-op. From b51d025ceb0918927440c3545ca604f3f64743e0 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 22:04:39 +0000 Subject: [PATCH 16/70] perf(component): move prepared rendered output into place Recreate each component's rendered output directory, then rename every prepared file and subdirectory into it except .git. Completely replace old output instead of copying the prepared tree. Preserve overwrite permission checks and check-only behavior. Dry runs leave both source and destination unchanged. Document that filesystem renames require the work directory and output to share a filesystem. Add in-memory tests covering new output, complete replacement, hidden and nested files, empty directories, excluded Git metadata, and safeguards. Validation: mage fix all passed. Build, unit tests, and default checks passed through mage all; Docker scenarios remain environment-blocked. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 11 ++- internal/app/azldev/cmds/component/render.go | 50 ++++++---- .../cmds/component/render_internal_test.go | 98 +++++++++++++++++++ 3 files changed, 137 insertions(+), 22 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 0d8c50477..d175e19f5 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -189,9 +189,14 @@ has no effect on lockless components. Synthetic history and its legacy release adjustment are skipped. The spec is **not** preprocessed with `rpmautospec process-distgit`: `%autorelease`, `%autochangelog`, and other RPM macros remain unexpanded. No mock configuration is needed for lockless rendering. -The renderer replaces the component output, excluding `.git`, -and commits only that directory. The subject is `updated `; the -body lists upstream changes, if any. Unrelated staged files are not committed. +After comparing the prepared tree with the current output, the renderer +recursively removes the existing component output directory, recreates it, and +moves all prepared files and subdirectories except `.git` into it. Files found +only in the old output are not retained. Moves use filesystem renames, so the +project work directory and rendered output must be on the same filesystem. +The renderer then commits only the component output directory. The subject is +`updated `; the body lists upstream changes, if any. Unrelated +staged files are not committed. Unchanged Git content does not produce an empty commit. A commit failure leaves the rendered files available for inspection. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 8524bdd74..a865a9f30 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -178,7 +178,7 @@ const ( // selected component into the output directory. Processing is done in four phases: // 1. Parallel source preparation (clone, overlay, synthetic git) // 2. Batch mock processing for locked components only -// 3. Parallel finishing (filter files, remove .git, copy output) +// 3. Parallel finishing (filter files, remove .git, move output) // 4. Serial project commits for lockless upstream component output func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, error) { if err := resolveAndValidateOutputDir(env, options); err != nil { @@ -241,7 +241,7 @@ func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, // Prune orphan component dirs (components removed from config) when // --clean-stale is set. Per-component output dirs that match the resolved // component set are NOT touched here; phase 3 will RemoveAll+rewrite each - // of them via copyRenderedOutput. This keeps the diff against existing + // of them via moveRenderedOutput. This keeps the diff against existing // output meaningful (so result.Changed reflects actual content drift // instead of being unconditionally true after a blanket wipe) and reduces // the blast radius of a Ctrl-C: we only ever delete dirs that wouldn't @@ -673,9 +673,9 @@ func batchMockProcess( // Phase 3: Parallel finishing // ────────────────────────────────────────────────────────────────────────────── -// parallelFinish applies source-query results (file filtering, .git removal) and copies +// parallelFinish applies source-query results (file filtering, .git removal) and moves // rendered output for all successfully prepared components. In --check-only -// mode, the copy step is replaced with a tree comparison and no disk writes +// mode, the move step is replaced with a tree comparison and no disk writes // happen. func parallelFinish( env *azldev.Env, @@ -843,9 +843,9 @@ func finishComponentRender( return changed, nil } - // Copy rendered files to the component's output directory. - if copyErr := copyRenderedOutput(env, componentDir, prep.compOutputDir, allowOverwrite); copyErr != nil { - return changed, copyErr + // Move rendered files to the component's output directory. + if moveErr := moveRenderedOutput(env, componentDir, prep.compOutputDir, allowOverwrite); moveErr != nil { + return changed, moveErr } // Best-effort: create a sibling symlink at the URL-encoded component name to @@ -908,10 +908,15 @@ func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName st return nil } -// copyRenderedOutput copies the rendered files from tempDir to the component's output directory. -// For managed output (inside project root), existing output is removed before copying. -// For external output, existing directories cause an error. -func copyRenderedOutput(env *azldev.Env, tempDir, componentOutputDir string, allowOverwrite bool) error { +// moveRenderedOutput recreates the output directory and moves all prepared entries +// except '.git' into it. Existing output requires allowOverwrite. +func moveRenderedOutput(env *azldev.Env, tempDir, componentOutputDir string, allowOverwrite bool) error { + if env.DryRun() { + slog.Info("Dry run; would replace rendered output", "source", tempDir, "dest", componentOutputDir) + + return nil + } + exists, existsErr := fileutils.DirExists(env.FS(), componentOutputDir) if existsErr != nil { return fmt.Errorf("checking output directory %#q:\n%w", componentOutputDir, existsErr) @@ -934,15 +939,22 @@ func copyRenderedOutput(env *azldev.Env, tempDir, componentOutputDir string, all return fmt.Errorf("creating output directory %#q:\n%w", componentOutputDir, mkdirErr) } - // Copy all files from temp to output. - copyOptions := fileutils.CopyDirOptions{ - CopyFileOptions: fileutils.CopyFileOptions{ - PreserveFileMode: true, - }, + entries, err := fileutils.ReadDir(env.FS(), tempDir) + if err != nil { + return fmt.Errorf("reading prepared output directory %#q:\n%w", tempDir, err) } - if copyErr := fileutils.CopyDirRecursive(env, env.FS(), tempDir, componentOutputDir, copyOptions); copyErr != nil { - return fmt.Errorf("copying rendered files to %#q:\n%w", componentOutputDir, copyErr) + for _, entry := range entries { + if entry.Name() == ".git" { + continue + } + + sourcePath := filepath.Join(tempDir, entry.Name()) + + destPath := filepath.Join(componentOutputDir, entry.Name()) + if moveErr := env.FS().Rename(sourcePath, destPath); moveErr != nil { + return fmt.Errorf("moving rendered entry %#q to %#q:\n%w", sourcePath, destPath, moveErr) + } } return nil @@ -1088,7 +1100,7 @@ func validateOutputDir(outputDir string) error { // pruneOrphanRenderedDirs removes per-component rendered-spec directories // under outputDir that don't correspond to any component in resolvedComps. // Per-component dirs that ARE in resolvedComps are left alone -- phase 3 will -// overwrite them via copyRenderedOutput, and leaving the prior content here +// overwrite them via moveRenderedOutput, and leaving the prior content here // lets the unconditional diff in finishComponentRender produce a meaningful // result.Changed value for the user-visible table. // diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 1a734eff8..0da1ae46b 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -8,6 +8,8 @@ import ( "path/filepath" "testing" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/spf13/afero" @@ -57,6 +59,102 @@ func TestFindSpecFile(t *testing.T) { }) } +func TestMoveRenderedOutput(t *testing.T) { + const ( + workingDir = "/working/curl" + outputDir = "/rendered/c/curl" + ) + + for _, testCase := range []struct { + name string + existing bool + allowOverwrite bool + dryRun bool + wantErr bool + }{ + {name: "new output"}, + {name: "replaces output", existing: true, allowOverwrite: true}, + {name: "requires overwrite permission", existing: true, wantErr: true}, + {name: "dry run preserves both trees", existing: true, allowOverwrite: true, dryRun: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + testEnv := testutils.NewTestEnv(t) + fs := testEnv.TestFS + env := azldev.NewEnv(t.Context(), azldev.EnvOptions{ + ProjectDir: testEnv.Env.ProjectDir(), Config: testEnv.Config, + Interfaces: testEnv.TestInterfaces, EventListener: testEnv.EventListener, + DryRunnable: azldev.NewAppDryRunnable(testCase.dryRun), + }) + + files := map[string]string{ + "curl.spec": "Name: curl\n", ".gitignore": "upstream ignore rules", + "extras/nested.patch": "nested sidecar", + } + for name, content := range files { + path := filepath.Join(workingDir, name) + require.NoError(t, fileutils.MkdirAll(fs, filepath.Dir(path))) + require.NoError(t, fileutils.WriteFile(fs, path, []byte(content), fileperms.PublicFile)) + } + + require.NoError(t, fileutils.MkdirAll(fs, filepath.Join(workingDir, "empty"))) + require.NoError(t, fileutils.MkdirAll(fs, filepath.Join(workingDir, ".git"))) + gitHead := filepath.Join(workingDir, ".git/HEAD") + require.NoError(t, fileutils.WriteFile(fs, gitHead, []byte("upstream history"), fileperms.PublicFile)) + + oldPath := filepath.Join(outputDir, "stale/nested.txt") + if testCase.existing { + require.NoError(t, fileutils.MkdirAll(fs, filepath.Dir(oldPath))) + require.NoError(t, fileutils.WriteFile(fs, oldPath, []byte("old output"), fileperms.PublicFile)) + } + + err := moveRenderedOutput(env, workingDir, outputDir, testCase.allowOverwrite) + if testCase.wantErr { + require.ErrorContains(t, err, "use --force to overwrite") + } else { + require.NoError(t, err) + } + + readDir := outputDir + if testCase.wantErr || testCase.dryRun { + readDir = workingDir + } + + for name, want := range files { + content, err := fileutils.ReadFile(fs, filepath.Join(readDir, name)) + require.NoError(t, err) + assert.Equal(t, want, string(content)) + } + + emptyExists, err := fileutils.DirExists(fs, filepath.Join(readDir, "empty")) + require.NoError(t, err) + assert.True(t, emptyExists) + + head, err := fileutils.ReadFile(fs, gitHead) + require.NoError(t, err) + assert.Equal(t, "upstream history", string(head)) + + if testCase.wantErr || testCase.dryRun { + old, err := fileutils.ReadFile(fs, oldPath) + require.NoError(t, err) + assert.Equal(t, "old output", string(old)) + + return + } + + remaining, err := fileutils.ReadDir(fs, workingDir) + require.NoError(t, err) + require.Len(t, remaining, 1) + assert.Equal(t, ".git", remaining[0].Name()) + + for _, path := range []string{oldPath, filepath.Join(outputDir, ".git")} { + exists, err := fileutils.Exists(fs, path) + require.NoError(t, err) + assert.False(t, exists) + } + }) + } +} + func TestWriteRenderErrorMarker(t *testing.T) { t.Run("creates marker file with static content", func(t *testing.T) { fs := afero.NewMemMapFs() From 00d7d73dfe3b91da62d47154453c25120fc2dfba Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 22:18:43 +0000 Subject: [PATCH 17/70] fix(component): apply overlays before release carryover Split lockless rendering around source preparation. Read committed render state and collect upstream changes first, then write macros, apply overlays, and update sources before release/changelog carryover and the final rpmdev-bumpspec invocation. Keep provenance capture on the pristine pinned checkout. Carryover and version-reset rules now operate on the overlaid working spec. Local and locked components retain their existing lifecycle. Add an in-memory pipeline regression proving macros, overlays, and source manifest updates precede carryover and bumping, and preserve macro-load directives when rewriting the spec. Update user and emitted guidance. Validation: mage fix all, mage unit, mage check all, and mage docs passed. Build passed through mage all; Docker scenarios remain environment-blocked. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 6 +- .../reference/cli/azldev_component_render.md | 4 +- .../app/azldev/agentskill/agentskill_test.go | 2 + .../azldev/agentskill/content/azldev.md.tmpl | 6 +- .../content/refresh-upstream-commit.md.tmpl | 9 +- .../content/withoutlockfile/azldev.md.tmpl | 5 +- internal/app/azldev/cmds/component/render.go | 27 ++++- .../azldev/cmds/component/render_lockless.go | 69 +++++++---- .../render_lockless_internal_test.go | 111 +++++++++++++++++- .../render_release_carryover_internal_test.go | 6 +- 10 files changed, 202 insertions(+), 43 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index d175e19f5..15b8e776b 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -182,7 +182,11 @@ It then always invokes host `rpmdev-bumpspec -c` with `updated ` followed by upstream changes on separate lines. The tool's no-op for `%autorelease` specs is accepted. -Existing overlays run after these steps. All prepared dist-git files are retained +Macro-file generation, existing overlays, and `sources` manifest updates run +before release/changelog carryover and the final `rpmdev-bumpspec` invocation. +Carryover and version-reset decisions therefore use the working spec after +overlays; release/changelog edits made by overlays remain subject to those rules. +All prepared dist-git files are retained except `.git`, including unreferenced sidecars, hidden files, and subdirectories. Lockless rendering does not run `spectool` or filter files; `render.skip-file-filter` has no effect on lockless components. Synthetic history and its legacy release diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 3ed1ff251..e376c9d20 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -26,8 +26,8 @@ its lock file, using configured upstream commits and their TOML history instead. All other components use lock files. A missing or empty file means all components use lock files. -For lockless upstream components, prepare release and changelog state from the -last committed render before applying the existing overlays and processing. +For lockless upstream components, write macros, apply overlays, and update sources +before carrying release and changelog state from the last committed render. Run rpmdev-bumpspec with "updated " and any upstream changes. After replacing the output, automatically commit that component directory to the project repository, without including unrelated staged files. The output diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 70d623d5c..bff9cecdb 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -179,6 +179,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "never uses mock or `rpmautospec process-distgit`") assert.Contains(t, doc, "all prepared dist-git files except `.git`") assert.NotContains(t, doc, "host `spectool` filtering") + assert.Contains(t, doc, "updates `sources` before carrying release/changelog state") } doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) @@ -193,6 +194,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "All prepared dist-git files except `.git` are retained") assert.Contains(t, doc, "immediately after checking out the pinned upstream commit") assert.Contains(t, doc, "that step never repeats the provenance query") + assert.Contains(t, doc, "the macros file, applies overlays, and updates `sources` before carrying") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 5f7fb5870..9b7493c3f 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -38,9 +38,9 @@ images assemble RPMs into bootable artifacts. workflow per component for all commands. Listed components ignore locks and use generated upstream-commit TOML instead. All others use locks. -For listed upstream components, rendering carries release/changelog state from -the committed output, runs host `rpmdev-bumpspec`, then applies the existing -overlays, without synthetic history or its legacy +For listed upstream components, rendering writes macros, applies overlays, and +updates `sources` before carrying release/changelog state from the committed +output and running host `rpmdev-bumpspec`, without synthetic history or its legacy release adjustment. Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. It retains all prepared dist-git files except `.git`, without file filtering or diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 6fcb22d86..445bd36de 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -68,6 +68,11 @@ runs here only for an upstream `%autorelease` whose `fedora_upstream_release` is not already supplied by `build.defines`. The captured values are reused when writing the macros file; that step never repeats the provenance query. +After reading render history and collecting upstream changes, rendering writes +the macros file, applies overlays, and updates `sources` before carrying +release/changelog state and running the final `rpmdev-bumpspec`. +Carryover and version-reset rules operate on the working spec after overlays. + For static specs, rendering checks the committed `Release:` value. If it uses the `baserelease` macro, rendering copies the committed `%global baserelease` or `%define baserelease` line over the upstream definition without changing @@ -84,8 +89,8 @@ an `%autorelease` upstream spec inherits the leading integer of the committed, macro-expanded Release as `-b` and its committed `%changelog` body as the working `changelog` file, removing leading and trailing comment lines starting with `##` while preserving interior comments. Expansion uses the committed files on the host; -the later version-change release reset still applies. Existing overlays -still run afterward. All prepared dist-git files except `.git` are retained, +the later version-change release reset still applies. +All prepared dist-git files except `.git` are retained, without file filtering or `spectool` queries. Synthetic history and its legacy release adjustment are skipped during lockless rendering. Lockless rendering never uses mock or `rpmautospec process-distgit`; the output diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 30298bd45..eaa1a0bde 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -33,8 +33,9 @@ images assemble RPMs into bootable artifacts. ## Common commands -For upstream components, rendering preserves committed downstream release and -changelog state before running host `rpmdev-bumpspec` and existing processing. +For upstream components, rendering writes macros, applies overlays, and +updates `sources` before carrying release/changelog state from the committed +output and running host `rpmdev-bumpspec`. It replaces and automatically commits only the component output directory. Commit refreshed TOML before rendering; configure Git author identity and install `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index a865a9f30..84295e8de 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -74,8 +74,8 @@ its lock file, using configured upstream commits and their TOML history instead. All other components use lock files. A missing or empty file means all components use lock files. -For lockless upstream components, prepare release and changelog state from the -last committed render before applying the existing overlays and processing. +For lockless upstream components, write macros, apply overlays, and update sources +before carrying release and changelog state from the last committed render. Run rpmdev-bumpspec with "updated " and any upstream changes. After replacing the output, automatically commit that component directory to the project repository, without including unrelated staged files. The output @@ -578,15 +578,17 @@ func prepareComponentSources( preparerOpts = append(preparerOpts, sources.WithMockProcessor(mockProcessor)) } - var update *locklessRenderUpdate + var ( + update *locklessRenderUpdate + history lastRender + ) if lockless && comp.GetConfig().Spec.SourceType == projectconfig.SpecSourceTypeUpstream { preparerOpts = append(preparerOpts, sources.WithFetchedSources( func(_ context.Context, fetched components.Component, dir string) error { var prepErr error //nolint:contextcheck // The callback context is the captured preparation env. - update, prepErr = prepareLocklessRender(env, hostRenderProcessor{env: env}, - fetched, dir, compOutputDir, specEditor) + history, update, prepErr = prepareLocklessRender(env, fetched, dir, compOutputDir, specEditor) return prepErr })) @@ -608,6 +610,21 @@ func prepareComponentSources( return nil, fmt.Errorf("preparing sources for %#q:\n%w", componentName, prepErr) } + return finishSourcePreparation(env, comp, componentDir, specEditor, history, update, lockless) +} + +func finishSourcePreparation( + env *azldev.Env, comp components.Component, componentDir string, specEditor spec.EditorMode, + history lastRender, update *locklessRenderUpdate, lockless bool, +) (*preparedComponent, error) { + componentName := comp.GetName() + if update != nil { + if err := prepareLocklessSpec(env, hostRenderProcessor{env: env}, componentDir, componentName, + specEditor, history, update); err != nil { + return nil, fmt.Errorf("preparing release and changelog for %#q:\n%w", componentName, err) + } + } + // Find the spec file so we can pass the filename to mock. specPath, specErr := findSpecFile(env.FS(), componentDir, componentName) if specErr != nil { diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 46aac794c..17a134027 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -54,35 +54,49 @@ type renderAutorelease interface { ExpandRelease(ctx context.Context, dir, filename string) (string, error) } +//nolint:gochecknoglobals // Replaced by isolated render tests to open an in-memory project repository. +var openRenderRepo = openChangedRepo + func prepareLocklessRender( - env *azldev.Env, autospec renderAutorelease, comp components.Component, + env *azldev.Env, comp components.Component, dir, outputDir string, editor spec.EditorMode, -) (*locklessRenderUpdate, error) { - repo, root, err := openChangedRepo(env) +) (history lastRender, update *locklessRenderUpdate, err error) { + repo, root, err := openRenderRepo(env) if err != nil { - return nil, err + return history, nil, err } path, err := repoRelPath(root, outputDir) if err != nil { - return nil, fmt.Errorf("lockless render output must be inside the project repository:\n%w", err) + return history, nil, fmt.Errorf("lockless render output must be inside the project repository:\n%w", err) } - history, err := readLastRender(env, repo, root, path, comp.GetName()) + history, err = readLastRender(env, repo, root, path, comp.GetName()) if err != nil { - return nil, fmt.Errorf("reading committed render state:\n%w", err) + return history, nil, fmt.Errorf("reading committed render state:\n%w", err) } - update := &locklessRenderUpdate{ + update = &locklessRenderUpdate{ repoRoot: root, path: path, subject: "updated " + comp.GetName(), } - if err := prepareLocklessSpec(env, autospec, dir, comp.GetName(), editor, history, update); err != nil { - return nil, err + + _, opened, err := openLocklessSpec(env, dir, comp.GetName(), editor) + if err != nil { + return history, nil, err } - return update, nil + version, err := opened.GetLastTag("", "Version") + if err != nil { + return history, nil, fmt.Errorf("reading upstream Version:\n%w", err) + } + + if err := collectRenderUpstreamChanges(env, dir, comp.GetName(), version, history, update); err != nil { + return history, nil, err + } + + return history, update, nil } func readLastRender( @@ -237,23 +251,34 @@ func lastRenderLockedUpstreamCommit(env *azldev.Env, name string) (string, error return lock.UpstreamCommit, nil } -func prepareLocklessSpec( - env *azldev.Env, autospec renderAutorelease, dir, name string, editor spec.EditorMode, - history lastRender, update *locklessRenderUpdate, -) error { - path, err := findSpecFile(env.FS(), dir, name) +func openLocklessSpec( + env *azldev.Env, dir, name string, editor spec.EditorMode, +) (path string, opened *spec.Spec, err error) { + path, err = findSpecFile(env.FS(), dir, name) if err != nil { - return err + return "", nil, err } content, err := fileutils.ReadFile(env.FS(), path) if err != nil { - return fmt.Errorf("reading upstream spec:\n%w", err) + return "", nil, fmt.Errorf("reading upstream spec:\n%w", err) } - opened, err := spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor)) + opened, err = spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor)) if err != nil { - return fmt.Errorf("parsing upstream spec:\n%w", err) + return "", nil, fmt.Errorf("parsing upstream spec:\n%w", err) + } + + return path, opened, nil +} + +func prepareLocklessSpec( + env *azldev.Env, autospec renderAutorelease, dir, name string, editor spec.EditorMode, + history lastRender, update *locklessRenderUpdate, +) error { + path, opened, err := openLocklessSpec(env, dir, name, editor) + if err != nil { + return err } original, err := opened.GetLastTag("", "Release") @@ -266,10 +291,6 @@ func prepareLocklessSpec( return fmt.Errorf("reading upstream Version:\n%w", err) } - if err := collectRenderUpstreamChanges(env, dir, name, version, history, update); err != nil { - return err - } - release, err := locklessRenderRelease(env, autospec, dir, filepath.Base(path), original, history) if err != nil { return err diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 9fa0a9f02..ca4ca866f 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -13,6 +13,8 @@ import ( "strings" "testing" + gogit "github.com/go-git/go-git/v5" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" @@ -441,6 +443,8 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { } stub := &renderAutospecStub{} update := &locklessRenderUpdate{subject: "updated curl"} + require.NoError(t, collectRenderUpstreamChanges(env.Env, "/working", "curl", + testCase.version, testCase.history, update)) err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, testCase.history, update) require.NoError(t, err) @@ -568,8 +572,10 @@ func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { return nil } - err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, - history, &locklessRenderUpdate{subject: "updated curl"}) + update := &locklessRenderUpdate{subject: "updated curl"} + require.NoError(t, collectRenderUpstreamChanges(env.Env, "/working", "curl", + testCase.version, history, update)) + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, history, update) require.NoError(t, err) assert.True(t, bumped) assert.Equal(t, []string{"expand"}, stub.calls) @@ -698,6 +704,107 @@ func TestPrepareComponentSources_LocalAndLockedSkipNewLifecycle(t *testing.T) { } } +func TestPrepareComponentSources_OverlaysBeforeReleaseCarryover(t *testing.T) { + env := testutils.NewTestEnv(t) + ctrl := gomock.NewController(t) + config := &projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + Build: projectconfig.ComponentBuildConfig{With: []string{"feature"}}, + Overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Release", Value: "2"}, + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Overlaid summary"}, + }, + SourceFiles: []projectconfig.SourceFileReference{{ + Filename: "extra-source.tar.gz", Hash: "abc123", HashType: fileutils.HashTypeSHA512, + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://example.com/extra-source.tar.gz"}, + }}, + } + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(config) + + const ( + oldChangelog = "* Downstream entry\n" + rootConfig = `[project] +default-distro = { name = "test-distro", version = "1.0" } +[distros.test-distro.versions."1.0"] +release-ver = "3" +dist-git-branch = "upstream" +[components.curl.spec] +type = "upstream" +upstream-commit = "aaaaaaa" +` + ) + + repo, _ := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + "azldev.toml": []byte(rootConfig), + "SPECS/c/curl/curl.spec": []byte(renderTestSpec("1", "9", oldChangelog)), + }}}) + originalOpen := openRenderRepo + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { + return repo, "/project", nil + } + manager := sourceproviders_test.NewMockSourceManager(ctrl) + manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile(env.TestFS, filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("1", "1", "* Upstream entry\n")), fileperms.PublicFile) + }) + + originalPreparer := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + t.Cleanup(func() { + openRenderRepo = originalOpen + newSourcePreparer = originalPreparer + }) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + bumped := false + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected upstream query: %v", cmd.Args) + + return "", nil + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + bumped = true + + assert.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) + path := cmd.Args[len(cmd.Args)-1] + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Contains(t, string(content), "Summary: Overlaid summary\n") + assert.Contains(t, string(content), "Release: 9\n") + assert.Contains(t, string(content), "%changelog\n"+oldChangelog) + assert.Contains(t, string(content), "%{load:%{_sourcedir}/curl.azl.macros}") + assert.Contains(t, string(content), "Source9999") + + macros, err := fileutils.ReadFile(env.TestFS, "/staging/curl/curl.azl.macros") + require.NoError(t, err) + assert.Contains(t, string(macros), "%_with_feature 1\n") + + manifest, err := fileutils.ReadFile(env.TestFS, "/staging/curl/sources") + require.NoError(t, err) + assert.Contains(t, string(manifest), "SHA512 (extra-source.tar.gz) = abc123") + + return fileutils.WriteFile(env.TestFS, path, + []byte(strings.Replace(string(content), "Release: 9\n", "Release: 10\n", 1)), fileperms.PublicFile) + } + _, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/project/SPECS/c/curl", + spec.EditorLegacy, false, true) + require.NoError(t, err) + assert.True(t, bumped) + + content, err := fileutils.ReadFile(env.TestFS, "/staging/curl/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: 10\n") +} + func TestPrepareComponentSources_LocklessSkipsSyntheticHistory(t *testing.T) { for _, release := range []string{"7%{?dist}", "%autorelease -b 9"} { t.Run(release, func(t *testing.T) { diff --git a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go index 557eb093e..bda8bc21f 100644 --- a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go +++ b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go @@ -94,8 +94,10 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { []byte(strings.Replace(string(content), "baserelease "+testCase.wantBase, "baserelease "+testCase.wantBumped, 1)), fileperms.PublicFile) } - err := prepareLocklessSpec(env.Env, &renderAutospecStub{}, "/working", "curl", editor, - history, &locklessRenderUpdate{subject: "updated curl"}) + update := &locklessRenderUpdate{subject: "updated curl"} + require.NoError(t, collectRenderUpstreamChanges(env.Env, "/working", "curl", + testCase.version, history, update)) + err := prepareLocklessSpec(env.Env, &renderAutospecStub{}, "/working", "curl", editor, history, update) require.NoError(t, err) assert.Equal(t, 1, calls) From b86b97e9518c4414af61fcc214df0b281c4cc55c Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 22:27:49 +0000 Subject: [PATCH 18/70] fix(component): ignore Git metadata during render comparison Keep the working checkout's .git entry through render comparison instead of removing it. Skip that root entry during directory traversal, so object databases are neither scanned nor compared. Continue excluding .git when moving prepared output. Add regression tests for normal and check-only rendering, unreadable ignored metadata, and payload paths that must still be compared. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 6 ++- internal/app/azldev/cmds/component/render.go | 21 +++----- .../cmds/component/render_internal_test.go | 47 ++++++++++++++++++ internal/utils/dirdiff/dirdiff.go | 28 ++++++----- internal/utils/dirdiff/dirdiff_test.go | 49 +++++++++++++++++++ internal/utils/dirdiff/types.go | 14 ++++++ 6 files changed, 138 insertions(+), 27 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 15b8e776b..ae31cfc3a 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -186,13 +186,15 @@ Macro-file generation, existing overlays, and `sources` manifest updates run before release/changelog carryover and the final `rpmdev-bumpspec` invocation. Carryover and version-reset decisions therefore use the working spec after overlays; release/changelog edits made by overlays remain subject to those rules. -All prepared dist-git files are retained -except `.git`, including unreferenced sidecars, hidden files, and subdirectories. +Rendered output retains all prepared dist-git files except `.git`, including +unreferenced sidecars, hidden files, and subdirectories. Lockless rendering does not run `spectool` or filter files; `render.skip-file-filter` has no effect on lockless components. Synthetic history and its legacy release adjustment are skipped. The spec is **not** preprocessed with `rpmautospec process-distgit`: `%autorelease`, `%autochangelog`, and other RPM macros remain unexpanded. No mock configuration is needed for lockless rendering. +Comparison ignores the root `.git` entry without traversing or removing it; +the working checkout retains its Git metadata until staging cleanup. After comparing the prepared tree with the current output, the renderer recursively removes the existing component output directory, recreates it, and moves all prepared files and subdirectories except `.git` into it. Files found diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 84295e8de..b3bd9ffbd 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -178,7 +178,7 @@ const ( // selected component into the output directory. Processing is done in four phases: // 1. Parallel source preparation (clone, overlay, synthetic git) // 2. Batch mock processing for locked components only -// 3. Parallel finishing (filter files, remove .git, move output) +// 3. Parallel finishing (filter locked files, compare without '.git', move output) // 4. Serial project commits for lockless upstream component output func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, error) { if err := resolveAndValidateOutputDir(env, options); err != nil { @@ -690,7 +690,7 @@ func batchMockProcess( // Phase 3: Parallel finishing // ────────────────────────────────────────────────────────────────────────────── -// parallelFinish applies source-query results (file filtering, .git removal) and moves +// parallelFinish applies source-query results, compares without '.git', and moves // rendered output for all successfully prepared components. In --check-only // mode, the move step is replaced with a tree comparison and no disk writes // happen. @@ -789,9 +789,9 @@ func finishOneComponent( return result } -// finishComponentRender applies mock results, filters unreferenced files, -// removes .git, diffs the staging tree against the existing on-disk output, -// and (unless checkOnly is set) copies the staging tree to the output dir. +// finishComponentRender applies mock results, filters locked components' files, +// diffs the staging tree against the existing output while ignoring the root '.git', +// and (unless checkOnly is set) moves all other entries to the output dir. // stagingDir is the root staging directory containing the component's subdirectory. // // Returns changed=true when the staging tree differs from the existing output @@ -841,13 +841,6 @@ func finishComponentRender( return false, fmt.Errorf("filtering unreferenced files for %#q:\n%w", componentName, filterErr) } - // Remove .git directory — must not appear in rendered output. - // rpmautospec already read it during the batch mock phase. - gitDir := filepath.Join(componentDir, ".git") - if removeErr := env.FS().RemoveAll(gitDir); removeErr != nil { - slog.Debug("Failed to remove .git directory", "path", gitDir, "error", removeErr) - } - // Compare staging tree to existing output. Always done so the result table // reflects which components actually changed on disk this run, not just // in --check-only mode. @@ -1229,7 +1222,7 @@ const renderErrorMarkerContent = "Rendering failed. See azldev logs for details. // the existing on-disk output (actualDir) and returns true when they differ. // A missing actualDir always counts as drift. Symlinks are compared by target // (filesystems without symlink support skip that check; matches production -// render behavior). +// render behavior). The root '.git' entry is ignored without traversing it. func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string) (bool, error) { actualExists, err := fileutils.DirExists(fileSystem, actualDir) if err != nil { @@ -1241,7 +1234,7 @@ func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string) (boo return true, nil } - result, err := dirdiff.DiffDirs(fileSystem, actualDir, expectedDir) + result, err := dirdiff.DiffDirs(fileSystem, actualDir, expectedDir, dirdiff.WithIgnoredPaths(".git")) if err != nil { return false, fmt.Errorf("diffing %#q vs %#q:\n%w", actualDir, expectedDir, err) } diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 0da1ae46b..6fb295c19 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -9,12 +9,15 @@ import ( "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" ) func TestFindSpecFile(t *testing.T) { @@ -695,6 +698,50 @@ func TestWriteFailureMarkers(t *testing.T) { }) } +func TestFinishComponentRender_PreservesGitMetadata(t *testing.T) { + for _, testCase := range []struct { + name string + checkOnly bool + }{ + {name: "normal render"}, + {name: "check-only", checkOnly: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + + for _, root := range []string{"/staging/curl", "/output/curl"} { + for _, name := range []string{"curl.spec", ".gitignore"} { + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(root, name), + []byte("unchanged"), fileperms.PublicFile)) + } + } + + const gitHead = "/staging/curl/.git/HEAD" + + require.NoError(t, fileutils.WriteFile(env.TestFS, gitHead, []byte("upstream"), fileperms.PublicFile)) + + prep := &preparedComponent{ + comp: comp, specFilename: "curl.spec", compOutputDir: "/output/curl", lockless: true, + } + mockResults := map[string]*sources.ComponentMockResult{"curl": {}} + + changed, err := finishComponentRender(env.Env, prep, mockResults, "/staging", true, testCase.checkOnly) + require.NoError(t, err) + assert.False(t, changed, "Git metadata alone must not count as rendered drift") + + head, err := fileutils.ReadFile(env.TestFS, gitHead) + require.NoError(t, err) + assert.Equal(t, "upstream", string(head)) + + exists, err := fileutils.Exists(env.TestFS, "/output/curl/.git") + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + func TestDiffRenderedOutput(t *testing.T) { t.Parallel() diff --git a/internal/utils/dirdiff/dirdiff.go b/internal/utils/dirdiff/dirdiff.go index ed2bb5360..06384d627 100644 --- a/internal/utils/dirdiff/dirdiff.go +++ b/internal/utils/dirdiff/dirdiff.go @@ -29,12 +29,12 @@ import ( func DiffDirs(fs opctx.FS, dirA, dirB string, opts ...DiffOption) (*DiffResult, error) { cfg := newDiffConfig(opts) - filesA, err := collectFiles(fs, dirA) + filesA, err := collectFiles(fs, dirA, cfg.ignoredPaths) if err != nil { return nil, fmt.Errorf("failed to walk original directory %#q:\n%w", dirA, err) } - filesB, err := collectFiles(fs, dirB) + filesB, err := collectFiles(fs, dirB, cfg.ignoredPaths) if err != nil { return nil, fmt.Errorf("failed to walk modified directory %#q:\n%w", dirB, err) } @@ -122,7 +122,7 @@ func mergedSortedKeys(filesA, filesB map[string]fileEntry) []string { // [afero.MemMapFs] does not support symlinks and will report them as regular files. // [afero.Walk] uses LstatIfPossible internally, so on supported filesystems symlinks // are correctly detected via [os.ModeSymlink] rather than being followed. -func collectFiles(fs opctx.FS, root string) (map[string]fileEntry, error) { +func collectFiles(fs opctx.FS, root string, ignoredPaths map[string]bool) (map[string]fileEntry, error) { result := make(map[string]fileEntry) walkErr := afero.Walk(fs, root, func(fullPath string, info os.FileInfo, err error) error { @@ -130,20 +130,26 @@ func collectFiles(fs opctx.FS, root string) (map[string]fileEntry, error) { return fmt.Errorf("error walking %#q:\n%w", fullPath, err) } - mode := info.Mode() + relPath, relErr := filepath.Rel(root, fullPath) + if relErr != nil { + return fmt.Errorf("failed to compute relative path for %#q from %#q:\n%w", fullPath, root, relErr) + } + + if ignoredPaths[relPath] { + if info.IsDir() { + return filepath.SkipDir + } - // Skip directories — we only collect leaf entries. Symlinks to directories - // are not reported as IsDir() by afero.Walk (which uses LstatIfPossible), - // so they correctly fall through to the symlink case below. - if info.IsDir() { return nil } - relPath, relErr := filepath.Rel(root, fullPath) - if relErr != nil { - return fmt.Errorf("failed to compute relative path for %#q from %#q:\n%w", fullPath, root, relErr) + // Walk reports symlinks to directories as symlinks, so only real directories are skipped. + if info.IsDir() { + return nil } + mode := info.Mode() + switch { case mode&os.ModeSymlink != 0: target, linkErr := readSymlinkTarget(fs, fullPath) diff --git a/internal/utils/dirdiff/dirdiff_test.go b/internal/utils/dirdiff/dirdiff_test.go index 4974eaa81..b64152158 100644 --- a/internal/utils/dirdiff/dirdiff_test.go +++ b/internal/utils/dirdiff/dirdiff_test.go @@ -15,6 +15,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/utils/dirdiff" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -32,6 +33,54 @@ func TestDiffDirs_IdenticalDirs(t *testing.T) { assert.Empty(t, result.String()) } +func TestDiffDirs_IgnoredPaths(t *testing.T) { + t.Parallel() + + for _, testCase := range []struct { + name string + gitPath string + }{ + {name: "directory", gitPath: ".git/HEAD"}, + {name: "file", gitPath: ".git"}, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + baseFS := afero.NewMemMapFs() + + for _, root := range []string{"/a", "/b"} { + for _, path := range []string{testCase.gitPath, ".gitignore", "nested/.git/HEAD"} { + require.NoError(t, fileutils.WriteFile(baseFS, filepath.Join(root, path), + []byte(root), fileperms.PublicFile)) + } + } + + testFS := unreadablePathFS{Fs: baseFS, blockedPath: "/a/.git"} + result, err := dirdiff.DiffDirs(testFS, "/a", "/b", dirdiff.WithIgnoredPaths(".git")) + require.NoError(t, err) + require.Len(t, result.Files, 2) + assert.Equal(t, ".gitignore", result.Files[0].Path) + assert.Equal(t, filepath.Join("nested", ".git", "HEAD"), result.Files[1].Path) + + _, err = dirdiff.DiffDirs(testFS, "/a", "/b") + require.ErrorIs(t, err, os.ErrPermission) + }) + } +} + +type unreadablePathFS struct { + afero.Fs + blockedPath string +} + +func (fs unreadablePathFS) Open(name string) (afero.File, error) { + if name == fs.blockedPath || strings.HasPrefix(name, fs.blockedPath+string(filepath.Separator)) { + return nil, os.ErrPermission + } + + return fs.Fs.Open(name) +} + func TestDiffDirs_ModifiedFile(t *testing.T) { ctx := testctx.NewCtx() testFS := ctx.FS() diff --git a/internal/utils/dirdiff/types.go b/internal/utils/dirdiff/types.go index 64a62f9b3..d44216bc3 100644 --- a/internal/utils/dirdiff/types.go +++ b/internal/utils/dirdiff/types.go @@ -3,6 +3,8 @@ package dirdiff +import "path/filepath" + // FileStatus describes whether a file was added, removed, modified, or changed type // relative to the original (pre-overlay) directory. type FileStatus string @@ -60,6 +62,7 @@ const ( type diffConfig struct { contextLines int maxBinaryScanSize int + ignoredPaths map[string]bool } func newDiffConfig(opts []DiffOption) *diffConfig { @@ -93,6 +96,17 @@ func WithMaxBinaryScanBytes(n int) DiffOption { } } +// WithIgnoredPaths excludes paths relative to each compared root. Ignored +// directories are not traversed. Each call replaces the previous ignored paths. +func WithIgnoredPaths(paths ...string) DiffOption { + return func(cfg *diffConfig) { + cfg.ignoredPaths = make(map[string]bool, len(paths)) + for _, path := range paths { + cfg.ignoredPaths[filepath.Clean(path)] = true + } + } +} + // fileKind categorizes directory entries for diffing purposes. type fileKind int From c380ced1ecb866b4fdd8b21531d9fa42ad0aa063 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 22:40:09 +0000 Subject: [PATCH 19/70] fix(component): compare lockless output before release bump Compare the overlaid, carried-forward tree before invoking the final rpmdev-bumpspec. Let check-only return at that comparison without requiring or running the final bump. Keep normal rendering's unconditional bump and include any spec changes from it in the reported result. Pass the worker context to the relocated command, and update ordering regressions, user documentation, and emitted agent guidance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 13 +- .../reference/cli/azldev_component_render.md | 4 +- .../app/azldev/agentskill/agentskill_test.go | 2 + .../azldev/agentskill/content/azldev.md.tmpl | 3 +- .../content/refresh-upstream-commit.md.tmpl | 7 +- .../content/withoutlockfile/azldev.md.tmpl | 1 + internal/app/azldev/cmds/component/render.go | 38 +++--- .../cmds/component/render_internal_test.go | 118 ++++++++++++++++++ .../azldev/cmds/component/render_lockless.go | 32 +++-- .../render_lockless_internal_test.go | 24 +++- .../render_release_carryover_internal_test.go | 4 + 11 files changed, 204 insertions(+), 42 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index ae31cfc3a..1db50de87 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -178,7 +178,9 @@ When upstream changes exist and `Version:` differs from the committed rendered version (or there is no previous output), it retains the original upstream `Release:` and `baserelease` definition instead of carrying over previous values, and restores the upstream `Release:` if an autorelease base was carried over. -It then always invokes host `rpmdev-bumpspec -c` with +It then compares the prepared tree with existing output, ignoring the root `.git`. +`--check-only` stops here, before the final release bump. +Normal rendering then always invokes host `rpmdev-bumpspec -c` with `updated ` followed by upstream changes on separate lines. The tool's no-op for `%autorelease` specs is accepted. @@ -195,7 +197,7 @@ adjustment are skipped. The spec is **not** preprocessed with `rpmautospec proce No mock configuration is needed for lockless rendering. Comparison ignores the root `.git` entry without traversing or removing it; the working checkout retains its Git metadata until staging cleanup. -After comparing the prepared tree with the current output, the renderer +After comparison and the final release bump, normal rendering recursively removes the existing component output directory, recreates it, and moves all prepared files and subdirectories except `.git` into it. Files found only in the old output are not retained. Moves use filesystem renames, so the @@ -217,9 +219,10 @@ Install `rpmdevtools`, `rpm-build`, and `rpmautospec` on the host as needed for the release/changelog queries, and configure Git author identity before rendering. Output (including a custom `--output-dir`) must be inside the project Git repository. `--check-only` prepares and compares temporary output but never -replaces rendered files or creates project commits. Because ordinary release -bumping is part of rendering, a new render may differ even without new upstream -commits. +performs the final `rpmdev-bumpspec` invocation, replaces rendered files, or +creates project commits. Normal rendering still performs its final bump even +when the pre-bump comparison finds no differences; its change reporting includes +changes made by that bump. ## Detect Changed Components diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index e376c9d20..4db76ea89 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -28,12 +28,12 @@ use lock files. For lockless upstream components, write macros, apply overlays, and update sources before carrying release and changelog state from the last committed render. -Run rpmdev-bumpspec with "updated " and any upstream changes. +Compare output, then run rpmdev-bumpspec with "updated " and upstream changes. After replacing the output, automatically commit that component directory to the project repository, without including unrelated staged files. The output must be inside that repository; Git author identity must be configured and rpmdev-bumpspec must be installed. Commit updated upstream-commit TOML before -rendering. --check-only never replaces output or creates project commits. +rendering. --check-only stops before bumping, replacing output, or committing. Lockless rendering runs entirely on the host: it never uses mock or rpmautospec process-distgit. The rendered spec retains its RPM macros, including %autorelease and %autochangelog. All prepared dist-git files are retained except diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index bff9cecdb..f81cef6e7 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -180,6 +180,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "all prepared dist-git files except `.git`") assert.NotContains(t, doc, "host `spectool` filtering") assert.Contains(t, doc, "updates `sources` before carrying release/changelog state") + assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") } doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) @@ -195,6 +196,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "immediately after checking out the pinned upstream commit") assert.Contains(t, doc, "that step never repeats the provenance query") assert.Contains(t, doc, "the macros file, applies overlays, and updates `sources` before carrying") + assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 9b7493c3f..f4df515a5 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -42,7 +42,8 @@ For listed upstream components, rendering writes macros, applies overlays, and updates `sources` before carrying release/changelog state from the committed output and running host `rpmdev-bumpspec`, without synthetic history or its legacy release adjustment. Lockless rendering -never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. +compares output before the final release bump; `--check-only` stops before that bump. +It never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. It retains all prepared dist-git files except `.git`, without file filtering or `spectool` queries, and replaces and automatically commits only that component's output directory; unrelated staged diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 445bd36de..cd6232412 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -72,6 +72,8 @@ After reading render history and collecting upstream changes, rendering writes the macros file, applies overlays, and updates `sources` before carrying release/changelog state and running the final `rpmdev-bumpspec`. Carryover and version-reset rules operate on the working spec after overlays. +Rendering compares output before the final release bump; `--check-only` stops before that bump. +The comparison ignores the root `.git` without traversing or removing it. For static specs, rendering checks the committed `Release:` value. If it uses the `baserelease` macro, rendering copies the committed `%global baserelease` @@ -107,9 +109,8 @@ automatically commits only that directory. The subject is `updated ` and the body contains upstream changes, if any. It never includes unrelated staged files or creates empty commits. Configure Git author identity and install host `rpmdevtools`, `rpm-build`, and `rpmautospec` -as needed. `--check-only` never replaces -output or commits, but a new static release bump may cause drift without new -upstream changes. +as needed. `--check-only` never replaces output or commits. Normal rendering +still performs its final bump when the pre-bump comparison finds no differences. For source-resolution changes, run `refresh-upstream-commit` first and review the generated pin, then follow the commit-before-render sequence above. Do not amend diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index eaa1a0bde..d4fdc4fad 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -36,6 +36,7 @@ images assemble RPMs into bootable artifacts. For upstream components, rendering writes macros, applies overlays, and updates `sources` before carrying release/changelog state from the committed output and running host `rpmdev-bumpspec`. +Rendering compares output before the final release bump; `--check-only` stops before that bump. It replaces and automatically commits only the component output directory. Commit refreshed TOML before rendering; configure Git author identity and install `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index b3bd9ffbd..c04684fe3 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -76,12 +76,12 @@ use lock files. For lockless upstream components, write macros, apply overlays, and update sources before carrying release and changelog state from the last committed render. -Run rpmdev-bumpspec with "updated " and any upstream changes. +Compare output, then run rpmdev-bumpspec with "updated " and upstream changes. After replacing the output, automatically commit that component directory to the project repository, without including unrelated staged files. The output must be inside that repository; Git author identity must be configured and rpmdev-bumpspec must be installed. Commit updated upstream-commit TOML before -rendering. --check-only never replaces output or creates project commits. +rendering. --check-only stops before bumping, replacing output, or committing. Lockless rendering runs entirely on the host: it never uses mock or rpmautospec process-distgit. The rendered spec retains its RPM macros, including %autorelease and %autochangelog. All prepared dist-git files are retained except @@ -690,10 +690,8 @@ func batchMockProcess( // Phase 3: Parallel finishing // ────────────────────────────────────────────────────────────────────────────── -// parallelFinish applies source-query results, compares without '.git', and moves -// rendered output for all successfully prepared components. In --check-only -// mode, the move step is replaced with a tree comparison and no disk writes -// happen. +// parallelFinish applies source-query results, compares without '.git', then +// bumps and moves rendered output. Check-only stops after comparison. func parallelFinish( env *azldev.Env, prepared []*preparedComponent, @@ -721,7 +719,8 @@ func parallelFinish( prepared, func(done, _ int) { progressEvent.SetProgress(int64(done), total) }, func(_ context.Context, prep *preparedComponent) *RenderResult { - return finishOneComponent(workerEnv, env, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly) + //nolint:contextcheck // workerEnv is the captured context supplied to parmap. + return finishOneComponent(workerEnv, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly) }, ) @@ -748,7 +747,6 @@ func parallelFinish( // worker; semaphore acquisition is handled by parmap. func finishOneComponent( workerEnv *azldev.Env, - env *azldev.Env, prep *preparedComponent, mockResultMap map[string]*sources.ComponentMockResult, stagingDir string, @@ -775,7 +773,7 @@ func finishOneComponent( Status: renderStatusOK, } - drifted, err := finishComponentRender(env, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly) + drifted, err := finishComponentRender(workerEnv, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly) if err != nil { slog.Error("Failed to finish rendering component", "component", componentName, "error", err) @@ -791,13 +789,12 @@ func finishOneComponent( // finishComponentRender applies mock results, filters locked components' files, // diffs the staging tree against the existing output while ignoring the root '.git', -// and (unless checkOnly is set) moves all other entries to the output dir. +// and (unless checkOnly is set) bumps lockless upstream specs and moves output. // stagingDir is the root staging directory containing the component's subdirectory. // -// Returns changed=true when the staging tree differs from the existing output -// (or no existing output is present). The diff is computed unconditionally so -// every render run gets a meaningful 'Changed' value in its result table; the -// disk write is the only thing gated by checkOnly. +// Returns changed=true when the prepared tree differs from the existing output +// (or no existing output is present), or a subsequent release bump changes the spec. +// Check-only compares before bumping and does not replace output. func finishComponentRender( env *azldev.Env, prep *preparedComponent, @@ -841,9 +838,7 @@ func finishComponentRender( return false, fmt.Errorf("filtering unreferenced files for %#q:\n%w", componentName, filterErr) } - // Compare staging tree to existing output. Always done so the result table - // reflects which components actually changed on disk this run, not just - // in --check-only mode. + // Compare before the final bump so '--check-only' does not manufacture release drift. changed, diffErr := diffRenderedOutput(env.FS(), componentDir, prep.compOutputDir) if diffErr != nil { return false, fmt.Errorf("comparing rendered output for %#q:\n%w", componentName, diffErr) @@ -853,6 +848,15 @@ func finishComponentRender( return changed, nil } + if prep.update != nil { + bumped, err := bumpLocklessSpec(env, specPath, prep.update) + if err != nil { + return changed, err + } + + changed = changed || bumped + } + // Move rendered files to the component's output directory. if moveErr := moveRenderedOutput(env, componentDir, prep.compOutputDir, allowOverwrite); moveErr != nil { return changed, moveErr diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 6fb295c19..9f80e605f 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -4,8 +4,11 @@ package component import ( + "errors" "os" + "os/exec" "path/filepath" + "strings" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" @@ -742,6 +745,121 @@ func TestFinishComponentRender_PreservesGitMetadata(t *testing.T) { } } +func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { + for _, testCase := range []struct { + name string + checkOnly bool + drift bool + autorelease bool + compareFail bool + wantChanged bool + }{ + {name: "check-only unchanged", checkOnly: true}, + {name: "check-only drift", checkOnly: true, drift: true, wantChanged: true}, + {name: "normal static bump", wantChanged: true}, + {name: "normal autorelease no-op", autorelease: true}, + {name: "normal drift and no-op", autorelease: true, drift: true, wantChanged: true}, + {name: "comparison fails", compareFail: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + baseFS := env.TestFS + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + + release := "9" + if testCase.autorelease { + release = "%autorelease" + } + + original := renderTestSpec("1", release, "* Previous entry\n") + + actual := original + if testCase.drift { + actual += "\n# Stale output\n" + } + + const ( + stagedSpec = "/staging/curl/curl.spec" + outputSpec = "/output/curl/curl.spec" + ) + + require.NoError(t, fileutils.WriteFile(baseFS, stagedSpec, []byte(original), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(baseFS, outputSpec, []byte(actual), fileperms.PublicFile)) + + comparisonFS := &renderComparisonFS{Fs: baseFS, path: outputSpec} + if testCase.compareFail { + comparisonFS.readErr = errors.New("comparison read failed") + } + + env.TestFS = comparisonFS + called := false + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + assert.True(t, comparisonFS.compared, "comparison must precede the bump") + + called = true + + assert.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) + + bumped := strings.Replace(original, "Release: 9\n", "Release: 10\n", 1) + + return fileutils.WriteFile(baseFS, stagedSpec, []byte(bumped), fileperms.PublicFile) + } + if !testCase.checkOnly { + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + } + + prep := &preparedComponent{ + comp: comp, specFilename: "curl.spec", compOutputDir: "/output/curl", + lockless: true, update: &locklessRenderUpdate{subject: "updated curl"}, + } + + changed, err := finishComponentRender(env.Env, prep, + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, testCase.checkOnly) + if testCase.compareFail { + require.ErrorIs(t, err, comparisonFS.readErr) + } else { + require.NoError(t, err) + } + + assert.True(t, comparisonFS.compared) + assert.Equal(t, !testCase.checkOnly && !testCase.compareFail, called) + assert.Equal(t, testCase.wantChanged, changed) + + want := strings.Replace(original, "Release: 9\n", "Release: 10\n", 1) + if testCase.checkOnly || testCase.compareFail { + want = actual + staged, readErr := fileutils.ReadFile(baseFS, stagedSpec) + require.NoError(t, readErr) + assert.Equal(t, original, string(staged)) + } + + output, readErr := fileutils.ReadFile(baseFS, outputSpec) + require.NoError(t, readErr) + assert.Equal(t, want, string(output)) + }) + } +} + +type renderComparisonFS struct { + afero.Fs + path string + compared bool + readErr error +} + +func (fs *renderComparisonFS) Open(name string) (afero.File, error) { + if name == fs.path { + fs.compared = true + if fs.readErr != nil { + return nil, fs.readErr + } + } + + return fs.Fs.Open(name) +} + func TestDiffRenderedOutput(t *testing.T) { t.Parallel() diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 17a134027..5285d6d58 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -318,12 +318,7 @@ func prepareLocklessSpec( return fmt.Errorf("writing prepared upstream spec:\n%w", err) } - entry := update.subject - if update.changes != "" { - entry += "\n" + update.changes - } - - return bumpLocklessSpec(env, path, entry) + return nil } func locklessSpecBytes(opened *spec.Spec, previous, release string, editor spec.EditorMode) ([]byte, error) { @@ -643,9 +638,19 @@ func renderChangelogBounds(content []byte, editor spec.EditorMode) (start int, e return start, end, nil } -func bumpLocklessSpec(env *azldev.Env, path, entry string) error { +func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate) (bool, error) { if err := prereqs.RequireExecutable(env, sources.RPMDevBumpspecBinary, nil); err != nil { - return fmt.Errorf("lockless upstream rendering requires 'rpmdev-bumpspec'; install 'rpmdevtools':\n%w", err) + return false, fmt.Errorf("lockless upstream rendering requires 'rpmdev-bumpspec'; install 'rpmdevtools':\n%w", err) + } + + before, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return false, fmt.Errorf("reading spec before release bump:\n%w", err) + } + + entry := update.subject + if update.changes != "" { + entry += "\n" + update.changes } raw := exec.CommandContext(env, sources.RPMDevBumpspecBinary, "-c", entry, path) @@ -657,14 +662,19 @@ func bumpLocklessSpec(env *azldev.Env, path, entry string) error { command, err := env.Command(raw) if err != nil { - return fmt.Errorf("creating render release bump command:\n%w", err) + return false, fmt.Errorf("creating render release bump command:\n%w", err) } if err := command.Run(env); err != nil { - return fmt.Errorf("bumping render release (%s):\n%w", stderr.String(), err) + return false, fmt.Errorf("bumping render release (%s):\n%w", stderr.String(), err) } - return nil + after, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return false, fmt.Errorf("reading spec after release bump:\n%w", err) + } + + return !bytes.Equal(before, after), nil } func commitLocklessRenders( diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index ca4ca866f..fd7032957 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -448,6 +448,10 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, testCase.history, update) require.NoError(t, err) + assert.False(t, called, "carryover must not perform the final bump") + + _, err = bumpLocklessSpec(env.Env, path, update) + require.NoError(t, err) assert.True(t, called) assert.Equal(t, testCase.wantChanges, update.changes) @@ -577,6 +581,10 @@ func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { testCase.version, history, update)) err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, history, update) require.NoError(t, err) + assert.False(t, bumped) + + _, err = bumpLocklessSpec(env.Env, "/working/curl.spec", update) + require.NoError(t, err) assert.True(t, bumped) assert.Equal(t, []string{"expand"}, stub.calls) assert.NotEqual(t, "/working", snapshot) @@ -795,12 +803,19 @@ upstream-commit = "aaaaaaa" return fileutils.WriteFile(env.TestFS, path, []byte(strings.Replace(string(content), "Release: 9\n", "Release: 10\n", 1)), fileperms.PublicFile) } - _, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/project/SPECS/c/curl", + prep, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/project/SPECS/c/curl", spec.EditorLegacy, false, true) require.NoError(t, err) + assert.False(t, bumped) + + prep.compOutputDir = "/project/SPECS/c/curl" + changed, err := finishComponentRender(env.Env, prep, + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, false) + require.NoError(t, err) + assert.True(t, changed) assert.True(t, bumped) - content, err := fileutils.ReadFile(env.TestFS, "/staging/curl/curl.spec") + content, err := fileutils.ReadFile(env.TestFS, "/project/SPECS/c/curl/curl.spec") require.NoError(t, err) assert.Contains(t, string(content), "Release: 10\n") } @@ -886,6 +901,9 @@ func TestBumpLocklessSpec_Failures(t *testing.T) { for _, available := range []bool{false, true} { t.Run(strconv.FormatBool(available), func(t *testing.T) { env := testutils.NewTestEnv(t) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(renderTestSpec("1", "1", "")), fileperms.PublicFile)) + if available { env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") } @@ -896,7 +914,7 @@ func TestBumpLocklessSpec_Failures(t *testing.T) { return errors.New("invalid spec") } - err := bumpLocklessSpec(env.Env, "/working/curl.spec", "updated curl") + _, err := bumpLocklessSpec(env.Env, "/working/curl.spec", &locklessRenderUpdate{subject: "updated curl"}) assert.Equal(t, available, called) diff --git a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go index bda8bc21f..8d4f4881d 100644 --- a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go +++ b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go @@ -99,6 +99,10 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { testCase.version, history, update)) err := prepareLocklessSpec(env.Env, &renderAutospecStub{}, "/working", "curl", editor, history, update) require.NoError(t, err) + assert.Zero(t, calls) + + _, err = bumpLocklessSpec(env.Env, path, update) + require.NoError(t, err) assert.Equal(t, 1, calls) content, err := fileutils.ReadFile(env.TestFS, path) From 401b33efc053d9205cb56a58ba788c88efbd6d31 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 23:03:52 +0000 Subject: [PATCH 20/70] feat(component): require explicit no-change rebuilds Reject unchanged lockless upstream renders unless --no-change-rebuild is set. Ignore .no-change-rebuild only in existing output when comparing, and record exact date -Is output in the working checkout before bumping an explicitly requested no-change rebuild. Preserve prior output on no-change refusals and return those errors even without --fail-on-error. Keep this policy scoped to lockless upstream renders; check-only remains read-only. Cover flag parsing, comparisons, forced rebuilds, and timestamp failures, and refresh CLI documentation and emitted agent guidance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 29 +++- .../reference/cli/azldev_component_render.md | 13 +- .../app/azldev/agentskill/agentskill_test.go | 6 + .../azldev/agentskill/content/azldev.md.tmpl | 4 + .../content/refresh-upstream-commit.md.tmpl | 7 +- .../content/withoutlockfile/azldev.md.tmpl | 4 + internal/app/azldev/cmds/component/render.go | 52 +++++-- .../cmds/component/render_internal_test.go | 128 +++++++++++++++--- .../azldev/cmds/component/render_lockless.go | 56 ++++++++ .../render_lockless_internal_test.go | 60 +++++++- .../app/azldev/cmds/component/render_test.go | 7 +- internal/utils/dirdiff/dirdiff.go | 7 +- internal/utils/dirdiff/dirdiff_test.go | 41 ++++++ internal/utils/dirdiff/types.go | 18 ++- 14 files changed, 382 insertions(+), 50 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 1db50de87..ef7b4306a 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -178,9 +178,26 @@ When upstream changes exist and `Version:` differs from the committed rendered version (or there is no previous output), it retains the original upstream `Release:` and `baserelease` definition instead of carrying over previous values, and restores the upstream `Release:` if an autorelease base was carried over. -It then compares the prepared tree with existing output, ignoring the root `.git`. +It then compares the prepared tree with existing output, ignoring the root `.git` +and any top-level `.no-change-rebuild` in the existing output only. `--check-only` stops here, before the final release bump. -Normal rendering then always invokes host `rpmdev-bumpspec -c` with +If normal rendering detects no changes, it returns an error indicating there +has been no change since the last render, without replacing the existing output +or writing a failure marker. To explicitly rebuild unchanged upstream content, +pass `--no-change-rebuild`, a boolean switch taking no argument: + +```bash +azldev comp render -p curl --no-change-rebuild +``` + +For an unchanged render with this flag, the tool runs host `date -Is` and writes +its exact output to `.no-change-rebuild` at the working dist-git root before +bumping. The marker is then published and committed with the rendered files. +The flag does not create a marker when comparison detects real changes. +A marker already present in the working upstream checkout remains part of the +comparison; only the existing output's marker is ignored. + +Changed renders, or explicitly requested no-change rebuilds, then invoke host `rpmdev-bumpspec -c` with `updated ` followed by upstream changes on separate lines. The tool's no-op for `%autorelease` specs is accepted. @@ -220,9 +237,11 @@ the release/changelog queries, and configure Git author identity before rendering. Output (including a custom `--output-dir`) must be inside the project Git repository. `--check-only` prepares and compares temporary output but never performs the final `rpmdev-bumpspec` invocation, replaces rendered files, or -creates project commits. Normal rendering still performs its final bump even -when the pre-bump comparison finds no differences; its change reporting includes -changes made by that bump. +creates project commits. `--check-only` also never creates a rebuild timestamp, +even with `--no-change-rebuild`. The no-change refusal and rebuild flag apply only +to lockless upstream components; local and locked components retain their existing +behavior. Change reporting includes a forced rebuild marker and changes made by +the final bump. ## Detect Changed Components diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 4db76ea89..ace1737a7 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -28,12 +28,12 @@ use lock files. For lockless upstream components, write macros, apply overlays, and update sources before carrying release and changelog state from the last committed render. -Compare output, then run rpmdev-bumpspec with "updated " and upstream changes. -After replacing the output, automatically commit that component directory to -the project repository, without including unrelated staged files. The output -must be inside that repository; Git author identity must be configured and -rpmdev-bumpspec must be installed. Commit updated upstream-commit TOML before -rendering. --check-only stops before bumping, replacing output, or committing. +Compare output; unchanged renders require --no-change-rebuild to record a timestamp. +Then run rpmdev-bumpspec with "updated " and upstream changes. +After replacing output, commit only that directory, excluding unrelated staged files. +Output must be inside the project repository; configure Git author identity and +install rpmdev-bumpspec. Commit updated upstream-commit TOML before rendering. +--check-only stops before bumping, replacing output, or committing. Lockless rendering runs entirely on the host: it never uses mock or rpmautospec process-distgit. The rendered spec retains its RPM macros, including %autorelease and %autochangelog. All prepared dist-git files are retained except @@ -83,6 +83,7 @@ azldev component render [flags] --fail-on-error exit with error if any component fails to render (useful for CI) -f, --force allow overwriting existing rendered component directories -h, --help help for render + --no-change-rebuild allow unchanged lockless upstream renders by recording 'date -Is' output in '.no-change-rebuild' -o, --output-dir string output directory for rendered specs (overrides rendered-specs-dir from config) --rpmdev-bumpspec Use rpmdev-bumpspec instead of the legacy static release calculation --skip-lock-validation skip lock file consistency checks diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index f81cef6e7..65680a829 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -181,6 +181,9 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.NotContains(t, doc, "host `spectool` filtering") assert.Contains(t, doc, "updates `sources` before carrying release/changelog state") assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") + assert.Contains(t, doc, "Unchanged lockless upstream renders require `--no-change-rebuild`") + assert.Contains(t, doc, "host `date -Is` output") + assert.Contains(t, doc, "ignores this marker in existing output, not in the working upstream checkout") } doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) @@ -197,6 +200,9 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "that step never repeats the provenance query") assert.Contains(t, doc, "the macros file, applies overlays, and updates `sources` before carrying") assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") + assert.Contains(t, doc, "Unchanged lockless upstream renders require `--no-change-rebuild`") + assert.Contains(t, doc, "host `date -Is` output") + assert.Contains(t, doc, "ignores this marker in existing output, not in the working upstream checkout") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index f4df515a5..81971ae6d 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -43,6 +43,10 @@ updates `sources` before carrying release/changelog state from the committed output and running host `rpmdev-bumpspec`, without synthetic history or its legacy release adjustment. Lockless rendering compares output before the final release bump; `--check-only` stops before that bump. +Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors +without replacing existing output. The flag records host `date -Is` output in +`.no-change-rebuild` before bumping, only when no changes were detected. Comparison +ignores this marker in existing output, not in the working upstream checkout. It never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. It retains all prepared dist-git files except `.git`, without file filtering or `spectool` queries, and replaces and diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index cd6232412..5d83a9c79 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -73,6 +73,10 @@ the macros file, applies overlays, and updates `sources` before carrying release/changelog state and running the final `rpmdev-bumpspec`. Carryover and version-reset rules operate on the working spec after overlays. Rendering compares output before the final release bump; `--check-only` stops before that bump. +Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors +without replacing existing output. The flag records host `date -Is` output in +`.no-change-rebuild` before bumping, only when no changes were detected. Comparison +ignores this marker in existing output, not in the working upstream checkout. The comparison ignores the root `.git` without traversing or removing it. For static specs, rendering checks the committed `Release:` value. If it uses @@ -109,8 +113,7 @@ automatically commits only that directory. The subject is `updated ` and the body contains upstream changes, if any. It never includes unrelated staged files or creates empty commits. Configure Git author identity and install host `rpmdevtools`, `rpm-build`, and `rpmautospec` -as needed. `--check-only` never replaces output or commits. Normal rendering -still performs its final bump when the pre-bump comparison finds no differences. +as needed. `--check-only` never replaces output, creates rebuild timestamps, or commits. For source-resolution changes, run `refresh-upstream-commit` first and review the generated pin, then follow the commit-before-render sequence above. Do not amend diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index d4fdc4fad..a79d561eb 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -37,6 +37,10 @@ For upstream components, rendering writes macros, applies overlays, and updates `sources` before carrying release/changelog state from the committed output and running host `rpmdev-bumpspec`. Rendering compares output before the final release bump; `--check-only` stops before that bump. +Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors +without replacing existing output. The flag records host `date -Is` output in +`.no-change-rebuild` before bumping, only when no changes were detected. Comparison +ignores this marker in existing output, not in the working upstream checkout. It replaces and automatically commits only the component output directory. Commit refreshed TOML before rendering; configure Git author identity and install `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index c04684fe3..2597e8956 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -38,6 +38,7 @@ type RenderOptions struct { Force bool CleanStale bool CheckOnly bool + NoChangeRebuild bool RPMDevBumpspec bool } @@ -76,12 +77,12 @@ use lock files. For lockless upstream components, write macros, apply overlays, and update sources before carrying release and changelog state from the last committed render. -Compare output, then run rpmdev-bumpspec with "updated " and upstream changes. -After replacing the output, automatically commit that component directory to -the project repository, without including unrelated staged files. The output -must be inside that repository; Git author identity must be configured and -rpmdev-bumpspec must be installed. Commit updated upstream-commit TOML before -rendering. --check-only stops before bumping, replacing output, or committing. +Compare output; unchanged renders require --no-change-rebuild to record a timestamp. +Then run rpmdev-bumpspec with "updated " and upstream changes. +After replacing output, commit only that directory, excluding unrelated staged files. +Output must be inside the project repository; configure Git author identity and +install rpmdev-bumpspec. Commit updated upstream-commit TOML before rendering. +--check-only stops before bumping, replacing output, or committing. Lockless rendering runs entirely on the host: it never uses mock or rpmautospec process-distgit. The rendered spec retains its RPM macros, including %autorelease and %autochangelog. All prepared dist-git files are retained except @@ -149,6 +150,8 @@ func addRenderOptions(cmd *cobra.Command, options *RenderOptions) { "without modifying the output directory. Exits 0 when nothing would change "+ "and 1 when any component would drift. With -a + --clean-stale, also fails "+ "on orphan rendered-spec directories. Intended for CI gates.") + cmd.Flags().BoolVar(&options.NoChangeRebuild, "no-change-rebuild", false, + "allow unchanged lockless upstream renders by recording 'date -Is' output in '.no-change-rebuild'") addRPMDevBumpspecFlag(cmd, &options.RPMDevBumpspec) // --check-only is a read-only diff against on-disk state; --fail-on-error @@ -165,6 +168,7 @@ type RenderResult struct { Status string `json:"status" table:"Status"` Error string `json:"error,omitempty" table:"Error,omitempty"` Changed bool `json:"changed" table:"Changed"` + renderErr error `json:"-" table:"-"` } // Render status constants. @@ -262,7 +266,7 @@ func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, // ── Phase 3: Parallel finishing ── parallelFinish(env, prepared, mockResultMap, results, stagingDir, - options.Force, options.CheckOnly) + options.Force, options.CheckOnly, options.NoChangeRebuild) // Write RENDER_FAILED markers for any component that errored in phase 1 // (source preparation) or phase 3 (mock result application + copy). @@ -356,15 +360,23 @@ func sortRenderResults(results []*RenderResult) { }) } -// checkRenderErrors counts error and cancelled results and returns an error if FailOnError is set. +// checkRenderErrors reports no-change refusals regardless of failOnError, and +// other error or cancelled results only when failOnError is set. func checkRenderErrors(results []*RenderResult, failOnError bool) error { - var errCount, cancelledCount int + var ( + errCount, cancelledCount int + noChangeErrors []error + ) for _, result := range results { if result == nil { continue } + if errors.Is(result.renderErr, errNoRenderChanges) { + noChangeErrors = append(noChangeErrors, fmt.Errorf("%#q: %w", result.Component, result.renderErr)) + } + switch result.Status { case renderStatusError: errCount++ @@ -381,6 +393,10 @@ func checkRenderErrors(results []*RenderResult, failOnError bool) error { slog.Error("Some components failed to render", "errorCount", errCount, "cancelledCount", cancelledCount) + if len(noChangeErrors) > 0 { + return errors.Join(noChangeErrors...) + } + if failOnError { return fmt.Errorf("%d component(s) failed to render", failCount) } @@ -700,6 +716,7 @@ func parallelFinish( stagingDir string, allowOverwrite bool, checkOnly bool, + noChangeRebuild bool, ) { if len(prepared) == 0 { return @@ -720,7 +737,7 @@ func parallelFinish( func(done, _ int) { progressEvent.SetProgress(int64(done), total) }, func(_ context.Context, prep *preparedComponent) *RenderResult { //nolint:contextcheck // workerEnv is the captured context supplied to parmap. - return finishOneComponent(workerEnv, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly) + return finishOneComponent(workerEnv, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly, noChangeRebuild) }, ) @@ -752,6 +769,7 @@ func finishOneComponent( stagingDir string, allowOverwrite bool, checkOnly bool, + noChangeRebuild bool, ) *RenderResult { componentName := prep.comp.GetName() compOutputDir := prep.compOutputDir @@ -773,13 +791,15 @@ func finishOneComponent( Status: renderStatusOK, } - drifted, err := finishComponentRender(workerEnv, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly) + drifted, err := finishComponentRender( + workerEnv, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly, noChangeRebuild) if err != nil { slog.Error("Failed to finish rendering component", "component", componentName, "error", err) result.Status = renderStatusError result.Error = err.Error() + result.renderErr = err } result.Changed = drifted @@ -802,6 +822,7 @@ func finishComponentRender( stagingDir string, allowOverwrite bool, checkOnly bool, + noChangeRebuild bool, ) (bool, error) { componentName := prep.comp.GetName() componentDir := filepath.Join(stagingDir, componentName) @@ -849,7 +870,7 @@ func finishComponentRender( } if prep.update != nil { - bumped, err := bumpLocklessSpec(env, specPath, prep.update) + bumped, err := finishLocklessSpec(env, specPath, prep.update, changed, noChangeRebuild) if err != nil { return changed, err } @@ -1150,6 +1171,7 @@ func pruneOrphanRenderedDirs( // Cancelled components are intentionally skipped — a Ctrl-C is not a render // failure, just an incomplete run, and silently planting markers under those // circumstances would lie in git diff. +// No-change refusals also leave the existing successful output intact. // // In --check-only mode, no marker is written. Instead, the existing on-disk // state for each errored component is verified to be exactly the standard @@ -1161,7 +1183,7 @@ func writeFailureMarkers( fileSystem opctx.FS, results []*RenderResult, allowOverwrite, checkOnly bool, ) { for _, result := range results { - if result == nil || result.Status != renderStatusError { + if result == nil || result.Status != renderStatusError || errors.Is(result.renderErr, errNoRenderChanges) { continue } @@ -1227,6 +1249,7 @@ const renderErrorMarkerContent = "Rendering failed. See azldev logs for details. // A missing actualDir always counts as drift. Symlinks are compared by target // (filesystems without symlink support skip that check; matches production // render behavior). The root '.git' entry is ignored without traversing it. +// The existing output's root '.no-change-rebuild' marker is also ignored. func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string) (bool, error) { actualExists, err := fileutils.DirExists(fileSystem, actualDir) if err != nil { @@ -1238,7 +1261,8 @@ func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string) (boo return true, nil } - result, err := dirdiff.DiffDirs(fileSystem, actualDir, expectedDir, dirdiff.WithIgnoredPaths(".git")) + result, err := dirdiff.DiffDirs(fileSystem, actualDir, expectedDir, + dirdiff.WithIgnoredPaths(".git"), dirdiff.WithIgnoredOriginalPaths(noChangeRebuildMarker)) if err != nil { return false, fmt.Errorf("diffing %#q vs %#q:\n%w", actualDir, expectedDir, err) } diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 9f80e605f..5d6a68f83 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -730,7 +730,7 @@ func TestFinishComponentRender_PreservesGitMetadata(t *testing.T) { } mockResults := map[string]*sources.ComponentMockResult{"curl": {}} - changed, err := finishComponentRender(env.Env, prep, mockResults, "/staging", true, testCase.checkOnly) + changed, err := finishComponentRender(env.Env, prep, mockResults, "/staging", true, testCase.checkOnly, false) require.NoError(t, err) assert.False(t, changed, "Git metadata alone must not count as rendered drift") @@ -747,18 +747,22 @@ func TestFinishComponentRender_PreservesGitMetadata(t *testing.T) { func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { for _, testCase := range []struct { - name string - checkOnly bool - drift bool - autorelease bool - compareFail bool - wantChanged bool + name string + checkOnly bool + drift bool + autorelease bool + compareFail bool + noChangeRebuild bool + wantNoChanges bool + wantChanged bool }{ - {name: "check-only unchanged", checkOnly: true}, + {name: "check-only unchanged with flag", checkOnly: true, noChangeRebuild: true}, {name: "check-only drift", checkOnly: true, drift: true, wantChanged: true}, - {name: "normal static bump", wantChanged: true}, - {name: "normal autorelease no-op", autorelease: true}, + {name: "unchanged render refused", wantNoChanges: true}, + {name: "forced static bump", noChangeRebuild: true, wantChanged: true}, + {name: "forced autorelease no-op", autorelease: true, noChangeRebuild: true, wantChanged: true}, {name: "normal drift and no-op", autorelease: true, drift: true, wantChanged: true}, + {name: "drift with flag", drift: true, noChangeRebuild: true, wantChanged: true}, {name: "comparison fails", compareFail: true}, } { t.Run(testCase.name, func(t *testing.T) { @@ -782,10 +786,13 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { const ( stagedSpec = "/staging/curl/curl.spec" outputSpec = "/output/curl/curl.spec" + timestamp = "2026-11-04T22:53:27+00:00\n" ) require.NoError(t, fileutils.WriteFile(baseFS, stagedSpec, []byte(original), fileperms.PublicFile)) require.NoError(t, fileutils.WriteFile(baseFS, outputSpec, []byte(actual), fileperms.PublicFile)) + markerPath := filepath.Join(filepath.Dir(outputSpec), noChangeRebuildMarker) + require.NoError(t, fileutils.WriteFile(baseFS, markerPath, []byte("old timestamp\n"), fileperms.PublicFile)) comparisonFS := &renderComparisonFS{Fs: baseFS, path: outputSpec} if testCase.compareFail { @@ -794,6 +801,9 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { env.TestFS = comparisonFS called := false + dated := false + wantBump := !testCase.checkOnly && !testCase.compareFail && !testCase.wantNoChanges + wantMarker := wantBump && !testCase.drift env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { assert.True(t, comparisonFS.compared, "comparison must precede the bump") @@ -802,12 +812,30 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { assert.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) + if wantMarker { + marker, err := fileutils.ReadFile(baseFS, filepath.Join(filepath.Dir(stagedSpec), noChangeRebuildMarker)) + require.NoError(t, err) + assert.Equal(t, timestamp, string(marker), "timestamp must be written before bumping") + } + bumped := strings.Replace(original, "Release: 9\n", "Release: 10\n", 1) return fileutils.WriteFile(baseFS, stagedSpec, []byte(bumped), fileperms.PublicFile) } + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.True(t, comparisonFS.compared) + assert.False(t, called) + assert.Equal(t, []string{"date", "-Is"}, cmd.Args) + assert.Equal(t, filepath.Dir(stagedSpec), cmd.Dir) + + dated = true + + return timestamp, nil + } if !testCase.checkOnly { env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + env.CmdFactory.RegisterCommandInSearchPath("date") } prep := &preparedComponent{ @@ -815,20 +843,32 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { lockless: true, update: &locklessRenderUpdate{subject: "updated curl"}, } - changed, err := finishComponentRender(env.Env, prep, - map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, testCase.checkOnly) - if testCase.compareFail { + result := finishOneComponent(env.Env, prep, + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, testCase.checkOnly, testCase.noChangeRebuild) + + err := result.renderErr + + switch { + case testCase.compareFail: require.ErrorIs(t, err, comparisonFS.readErr) - } else { + case testCase.wantNoChanges: + require.ErrorIs(t, err, errNoRenderChanges) + writeFailureMarkers(baseFS, []*RenderResult{result}, true, false) + + for _, failOnError := range []bool{false, true} { + require.ErrorIs(t, checkRenderErrors([]*RenderResult{result}, failOnError), errNoRenderChanges) + } + default: require.NoError(t, err) } assert.True(t, comparisonFS.compared) - assert.Equal(t, !testCase.checkOnly && !testCase.compareFail, called) - assert.Equal(t, testCase.wantChanged, changed) + assert.Equal(t, wantBump, called) + assert.Equal(t, wantMarker, dated) + assert.Equal(t, testCase.wantChanged, result.Changed) want := strings.Replace(original, "Release: 9\n", "Release: 10\n", 1) - if testCase.checkOnly || testCase.compareFail { + if !wantBump { want = actual staged, readErr := fileutils.ReadFile(baseFS, stagedSpec) require.NoError(t, readErr) @@ -838,6 +878,22 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { output, readErr := fileutils.ReadFile(baseFS, outputSpec) require.NoError(t, readErr) assert.Equal(t, want, string(output)) + + markerExists, readErr := fileutils.Exists(baseFS, markerPath) + require.NoError(t, readErr) + assert.Equal(t, !wantBump || wantMarker, markerExists) + + if markerExists { + marker, readErr := fileutils.ReadFile(baseFS, markerPath) + require.NoError(t, readErr) + + wantTimestamp := "old timestamp\n" + if wantMarker { + wantTimestamp = timestamp + } + + assert.Equal(t, wantTimestamp, string(marker)) + } }) } } @@ -914,6 +970,44 @@ func TestDiffRenderedOutput(t *testing.T) { }) } +func TestDiffRenderedOutput_RebuildMarker(t *testing.T) { + t.Parallel() + + for _, testCase := range []struct { + name, existing, working string + wantChanged bool + }{ + {name: "existing marker ignored", existing: noChangeRebuildMarker}, + {name: "working marker compared", working: noChangeRebuildMarker, wantChanged: true}, + {name: "both markers", existing: noChangeRebuildMarker, working: noChangeRebuildMarker, wantChanged: true}, + {name: "nested marker compared", existing: "nested/" + noChangeRebuildMarker, wantChanged: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + testFS := afero.NewMemMapFs() + for _, root := range []string{"/existing", "/working"} { + require.NoError(t, fileutils.WriteFile(testFS, filepath.Join(root, "curl.spec"), + []byte("identical"), fileperms.PublicFile)) + } + + if testCase.existing != "" { + require.NoError(t, fileutils.WriteFile(testFS, filepath.Join("/existing", testCase.existing), + []byte("timestamp\n"), fileperms.PublicFile)) + } + + if testCase.working != "" { + require.NoError(t, fileutils.WriteFile(testFS, filepath.Join("/working", testCase.working), + []byte("timestamp\n"), fileperms.PublicFile)) + } + + changed, err := diffRenderedOutput(testFS, "/working", "/existing") + require.NoError(t, err) + assert.Equal(t, testCase.wantChanged, changed) + }) + } +} + func TestOutputDriftsFromMarker(t *testing.T) { t.Parallel() diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 5285d6d58..f1cbd484f 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -32,6 +32,10 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" ) +const noChangeRebuildMarker = ".no-change-rebuild" + +var errNoRenderChanges = errors.New("no change since the last render; use '--no-change-rebuild' to rebuild anyway") + type locklessRenderUpdate struct { repoRoot string path string @@ -638,6 +642,58 @@ func renderChangelogBounds(content []byte, editor spec.EditorMode) (start int, e return start, end, nil } +func finishLocklessSpec( + env *azldev.Env, path string, update *locklessRenderUpdate, changed, noChangeRebuild bool, +) (bool, error) { + if !changed { + if !noChangeRebuild { + return false, errNoRenderChanges + } + + if err := writeNoChangeRebuild(env, filepath.Dir(path)); err != nil { + return false, err + } + } + + bumped, err := bumpLocklessSpec(env, path, update) + + return !changed || bumped, err +} + +func writeNoChangeRebuild(env *azldev.Env, dir string) error { + if env.DryRun() { + return nil + } + + if err := prereqs.RequireExecutable(env, "date", nil); err != nil { + return fmt.Errorf("no-change rebuild requires 'date':\n%w", err) + } + + raw := exec.CommandContext(env, "date", "-Is") + raw.Dir = dir + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + command, err := env.Command(raw) + if err != nil { + return fmt.Errorf("creating rebuild timestamp command:\n%w", err) + } + + output, err := command.RunAndGetOutput(env) + if err != nil { + return fmt.Errorf("getting rebuild timestamp (%s):\n%w", stderr.String(), err) + } + + if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, noChangeRebuildMarker), + []byte(output), fileperms.PublicFile); err != nil { + return fmt.Errorf("writing rebuild timestamp:\n%w", err) + } + + return nil +} + func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate) (bool, error) { if err := prereqs.RequireExecutable(env, sources.RPMDevBumpspecBinary, nil); err != nil { return false, fmt.Errorf("lockless upstream rendering requires 'rpmdev-bumpspec'; install 'rpmdevtools':\n%w", err) diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index fd7032957..6d7a3843e 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -26,6 +26,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -810,7 +811,7 @@ upstream-commit = "aaaaaaa" prep.compOutputDir = "/project/SPECS/c/curl" changed, err := finishComponentRender(env.Env, prep, - map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, false) + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, false, false) require.NoError(t, err) assert.True(t, changed) assert.True(t, bumped) @@ -897,6 +898,63 @@ func TestPrepareLocklessSpec_StopsOnAutospecFailure(t *testing.T) { assert.Equal(t, []string{"changelog"}, stub.calls) } +func TestFinishLocklessSpec_TimestampFailures(t *testing.T) { + for _, testCase := range []struct { + name, want string + available bool + dateFails bool + readOnly bool + }{ + {name: "date missing", want: "requires 'date'"}, + {name: "date fails", available: true, dateFails: true, want: "getting rebuild timestamp"}, + {name: "write fails", available: true, readOnly: true, want: "writing rebuild timestamp"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + const path = "/working/curl.spec" + + original := renderTestSpec("1", "9", "") + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(original), fileperms.PublicFile)) + + if testCase.readOnly { + env.TestFS = afero.NewReadOnlyFs(env.TestFS) + } + + if testCase.available { + env.CmdFactory.RegisterCommandInSearchPath("date") + } + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("must not bump without a timestamp: %v", cmd.Args) + + return nil + } + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{"date", "-Is"}, cmd.Args) + + if testCase.dateFails { + return "", errors.New("date failed") + } + + return "2026-11-04T22:53:27+00:00\n", nil + } + + changed, err := finishLocklessSpec(env.Env, path, &locklessRenderUpdate{}, false, true) + require.ErrorContains(t, err, testCase.want) + assert.False(t, changed) + + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Equal(t, original, string(content)) + + exists, err := fileutils.Exists(env.TestFS, filepath.Join("/working", noChangeRebuildMarker)) + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + func TestBumpLocklessSpec_Failures(t *testing.T) { for _, available := range []bool{false, true} { t.Run(strconv.FormatBool(available), func(t *testing.T) { diff --git a/internal/app/azldev/cmds/component/render_test.go b/internal/app/azldev/cmds/component/render_test.go index 00a03def8..aadae63d1 100644 --- a/internal/app/azldev/cmds/component/render_test.go +++ b/internal/app/azldev/cmds/component/render_test.go @@ -49,6 +49,11 @@ func TestNewRenderCmd_Flags(t *testing.T) { require.NotNil(t, checkOnlyFlag, "check-only flag should be registered") assert.Equal(t, "false", checkOnlyFlag.DefValue) + noChangeRebuildFlag := cmd.Flags().Lookup("no-change-rebuild") + require.NotNil(t, noChangeRebuildFlag) + require.NoError(t, cmd.ParseFlags([]string{"--no-change-rebuild"})) + assert.Equal(t, "true", noChangeRebuildFlag.Value.String()) + rpmDevBumpspecFlag := cmd.Flags().Lookup("rpmdev-bumpspec") require.NotNil(t, rpmDevBumpspecFlag) assert.Equal(t, "false", rpmDevBumpspecFlag.DefValue) @@ -61,7 +66,7 @@ func TestRenderCmd_NoComponents(t *testing.T) { testEnv := testutils.NewTestEnv(t) cmd := componentcmds.NewRenderCmd() - cmd.SetArgs([]string{"-o", "SPECS", "nonexistent-component"}) + cmd.SetArgs([]string{"-o", "SPECS", "--no-change-rebuild", "nonexistent-component"}) err := cmd.ExecuteContext(testEnv.Env) diff --git a/internal/utils/dirdiff/dirdiff.go b/internal/utils/dirdiff/dirdiff.go index 06384d627..3dd728109 100644 --- a/internal/utils/dirdiff/dirdiff.go +++ b/internal/utils/dirdiff/dirdiff.go @@ -7,6 +7,7 @@ package dirdiff import ( "bytes" "fmt" + "maps" "os" "path/filepath" "slices" @@ -29,7 +30,11 @@ import ( func DiffDirs(fs opctx.FS, dirA, dirB string, opts ...DiffOption) (*DiffResult, error) { cfg := newDiffConfig(opts) - filesA, err := collectFiles(fs, dirA, cfg.ignoredPaths) + ignoredOriginal := make(map[string]bool, len(cfg.ignoredPaths)+len(cfg.ignoredOriginalPaths)) + maps.Copy(ignoredOriginal, cfg.ignoredPaths) + maps.Copy(ignoredOriginal, cfg.ignoredOriginalPaths) + + filesA, err := collectFiles(fs, dirA, ignoredOriginal) if err != nil { return nil, fmt.Errorf("failed to walk original directory %#q:\n%w", dirA, err) } diff --git a/internal/utils/dirdiff/dirdiff_test.go b/internal/utils/dirdiff/dirdiff_test.go index b64152158..adeffcd15 100644 --- a/internal/utils/dirdiff/dirdiff_test.go +++ b/internal/utils/dirdiff/dirdiff_test.go @@ -81,6 +81,47 @@ func (fs unreadablePathFS) Open(name string) (afero.File, error) { return fs.Fs.Open(name) } +func TestDiffDirs_IgnoredOriginalPaths(t *testing.T) { + t.Parallel() + + for _, testCase := range []struct { + name string + inModified bool + }{ + {name: "original only"}, + {name: "modified entry is still compared", inModified: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + baseFS := afero.NewMemMapFs() + for _, root := range []string{"/a", "/b"} { + require.NoError(t, fileutils.WriteFile(baseFS, filepath.Join(root, ".git/HEAD"), + []byte(root), fileperms.PublicFile)) + } + + require.NoError(t, fileutils.WriteFile(baseFS, "/a/marker", []byte("original"), fileperms.PublicFile)) + + if testCase.inModified { + require.NoError(t, fileutils.WriteFile(baseFS, "/b/marker", []byte("modified"), fileperms.PublicFile)) + } + + fs := unreadablePathFS{Fs: baseFS, blockedPath: "/a/marker"} + result, err := dirdiff.DiffDirs(fs, "/a", "/b", + dirdiff.WithIgnoredPaths(".git"), dirdiff.WithIgnoredOriginalPaths("marker")) + require.NoError(t, err) + + if testCase.inModified { + require.Len(t, result.Files, 1) + assert.Equal(t, "marker", result.Files[0].Path) + assert.Equal(t, dirdiff.FileStatusAdded, result.Files[0].Status) + } else { + assert.Empty(t, result.Files) + } + }) + } +} + func TestDiffDirs_ModifiedFile(t *testing.T) { ctx := testctx.NewCtx() testFS := ctx.FS() diff --git a/internal/utils/dirdiff/types.go b/internal/utils/dirdiff/types.go index d44216bc3..4ae547c96 100644 --- a/internal/utils/dirdiff/types.go +++ b/internal/utils/dirdiff/types.go @@ -60,9 +60,10 @@ const ( // diffConfig holds the resolved configuration for a [DiffDirs] invocation. type diffConfig struct { - contextLines int - maxBinaryScanSize int - ignoredPaths map[string]bool + contextLines int + maxBinaryScanSize int + ignoredPaths map[string]bool + ignoredOriginalPaths map[string]bool } func newDiffConfig(opts []DiffOption) *diffConfig { @@ -107,6 +108,17 @@ func WithIgnoredPaths(paths ...string) DiffOption { } } +// WithIgnoredOriginalPaths additionally excludes root-relative paths only from +// the original (dirA) tree. Matching entries in the modified tree are still compared. +func WithIgnoredOriginalPaths(paths ...string) DiffOption { + return func(cfg *diffConfig) { + cfg.ignoredOriginalPaths = make(map[string]bool, len(paths)) + for _, path := range paths { + cfg.ignoredOriginalPaths[filepath.Clean(path)] = true + } + } +} + // fileKind categorizes directory entries for diffing purposes. type fileKind int From db09526729e012b8f3918d1def3ef502c4163781 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 4 Nov 2026 23:40:21 +0000 Subject: [PATCH 21/70] perf(component): use native Git for render history Find the last component render with a path-scoped native git log instead of go-git's project-wide tree diffs. Pin the query to the already-resolved HEAD and treat the directory path literally. Keep committed spec and configuration loading unchanged. Cover the command arguments, empty results, subprocess errors and missing objects with in-memory tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- .../azldev/cmds/component/render_lockless.go | 14 +--- .../render_lockless_internal_test.go | 79 +++++++++++++++++-- 2 files changed, 77 insertions(+), 16 deletions(-) diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index f1cbd484f..f6a1876d8 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "io" "log/slog" "os/exec" "path/filepath" @@ -141,22 +140,17 @@ func readLastRender( return history, fmt.Errorf("reading committed spec contents:\n%w", err) } - log, err := repo.Log(&gogit.LogOptions{ - From: head.Hash(), - PathFilter: func(file string) bool { - return strings.HasPrefix(file, path+"/") - }, - }) + lastHash, err := git.RunInDir(env, env, root, + "--literal-pathspecs", "log", "-1", "--format=%H", "--end-of-options", head.Hash().String(), "--", path+"/") if err != nil { return history, fmt.Errorf("finding last render commit:\n%w", err) } - defer log.Close() - last, err := log.Next() - if errors.Is(err, io.EOF) { + if lastHash == "" { return history, nil } + last, err := repo.CommitObject(plumbing.NewHash(lastHash)) if err != nil { return history, fmt.Errorf("reading last render commit:\n%w", err) } diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 6d7a3843e..d412c5828 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -53,7 +53,7 @@ release-ver = "3" dist-git-branch = "f42" `) oldSpec := renderTestSpec("1", "9%{?dist}", "* Previous\n- Keep exactly\n") - repo, _ := testRepoWithCommits(t, []testRepoCommit{ + repo, hashes := testRepoWithCommits(t, []testRepoCommit{ {files: map[string][]byte{ "azldev.toml": root, "component.toml": []byte("[components.curl.spec]\ntype = 'upstream'\n" + @@ -77,6 +77,14 @@ dist-git-branch = "f42" require.NoError(t, dirty.Close()) env := testutils.NewTestEnv(t) + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{ + "git", "-C", "/project", "--literal-pathspecs", "log", "-1", "--format=%H", + "--end-of-options", hashes[2], "--", "SPECS/c/curl/", + }, cmd.Args) + + return hashes[0] + "\n", nil + } history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl") require.NoError(t, err) assert.Equal(t, oldSpec, history.spec) @@ -92,6 +100,56 @@ dist-git-branch = "f42" assert.Equal(t, "f41", history.currentBranch) } +func TestReadLastRender_NativeHistoryErrors(t *testing.T) { + const renderPath = "SPECS/c/curl[1]" + + logErr := errors.New("git log failed") + + for _, testCase := range []struct { + name, output, wantError string + commandErr error + }{ + {name: "no matching commit"}, + {name: "git failure", commandErr: logErr, wantError: "finding last render commit"}, + {name: "unknown commit", output: strings.Repeat("a", 40), wantError: "reading last render commit"}, + } { + t.Run(testCase.name, func(t *testing.T) { + previous := renderTestSpec("1", "9", "old\n") + repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + renderPath + "/curl.spec": []byte(previous), + }}}) + env := testutils.NewTestEnv(t) + called := false + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + called = true + + assert.Equal(t, []string{ + "git", "-C", "/project", "--literal-pathspecs", "log", "-1", "--format=%H", + "--end-of-options", hashes[0], "--", renderPath + "/", + }, cmd.Args) + + return testCase.output, testCase.commandErr + } + + history, err := readLastRender(env.Env, repo, "/project", renderPath, "curl") + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + } else { + require.NoError(t, err) + } + + if testCase.commandErr != nil { + require.ErrorIs(t, err, testCase.commandErr) + } + + assert.True(t, called) + assert.Equal(t, previous, history.spec) + assert.Empty(t, history.previous) + assert.Empty(t, history.current) + }) + } +} + func TestReadLastRender_HistoricalLockFallback(t *testing.T) { const ( oldLock = "version = 1\nupstream-commit = 'aaaaaaa'\n" @@ -152,7 +210,7 @@ dist-git-branch = "f40" selection = "curl\n" } - repo, _ := testRepoWithCommits(t, []testRepoCommit{ + repo, hashes := testRepoWithCommits(t, []testRepoCommit{ {files: files}, {files: map[string][]byte{ "components.toml": []byte(current), @@ -171,6 +229,9 @@ dist-git-branch = "f40" require.NoError(t, err) require.NoError(t, dirty.Close()) env := testutils.NewTestEnv(t) + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return hashes[0], nil + } history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl") if testCase.missingCurrent { @@ -307,8 +368,11 @@ upstream-commit = "bbbbbbb" commits = append(commits, testRepoCommit{files: map[string][]byte{"azldev.toml": testCase.current}}) } - repo, _ := testRepoWithCommits(t, commits) + repo, hashes := testRepoWithCommits(t, commits) env := testutils.NewTestEnv(t) + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return hashes[0], nil + } history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl") if testCase.wantError != "" { @@ -745,7 +809,7 @@ upstream-commit = "aaaaaaa" ` ) - repo, _ := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ "azldev.toml": []byte(rootConfig), "SPECS/c/curl/curl.spec": []byte(renderTestSpec("1", "9", oldChangelog)), }}}) @@ -776,9 +840,12 @@ upstream-commit = "aaaaaaa" bumped := false env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { - t.Fatalf("unexpected upstream query: %v", cmd.Args) + assert.Equal(t, []string{ + "git", "-C", "/project", "--literal-pathspecs", "log", "-1", "--format=%H", + "--end-of-options", hashes[0], "--", "SPECS/c/curl/", + }, cmd.Args) - return "", nil + return hashes[0], nil } env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { bumped = true From a68f1c928bcde027f26b69e64ea79c26e22d25a9 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 00:01:51 +0000 Subject: [PATCH 22/70] fix(component): preserve committed autorelease state Carry the committed -b option into an unprocessed autorelease spec and replace the working changelog with its committed contents. Use base 1 when the option is omitted, without running host release queries. Retain legacy-marker migration and version-change reset behavior. Add in-memory regressions and update the user guide and emitted agent skill. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 8 ++ .../app/azldev/agentskill/agentskill_test.go | 2 + .../content/refresh-upstream-commit.md.tmpl | 5 + .../azldev/cmds/component/render_lockless.go | 50 ++++++- .../render_lockless_internal_test.go | 136 +++++++++++++++++- 5 files changed, 199 insertions(+), 2 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index ef7b4306a..36a6b28b8 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -156,6 +156,14 @@ leading or trailing comment lines starting with `##`. Interior comments and changelog whitespace are preserved. The later version-change release reset still applies. +For an existing `%autorelease` spec without the legacy marker, rendering carries +the committed `-b` value into the working spec, preserving the working macro's +other options. An omitted `-b` means the default base of `1`. The committed +`changelog` file replaces the working file byte-for-byte, or is created if absent +from the working checkout. A missing committed `changelog` or invalid numeric +base fails rendering. No host release or changelog query is needed for this +carryover; the later version-change release reset still applies. + The last commit touching the component output is the **last render commit**. The renderer fully loads TOML, including includes and component defaults, at that commit and at the current project commit to compare `spec.upstream-commit`. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 65680a829..72cf3d998 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -192,6 +192,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "automatically commits only that directory") assert.Contains(t, doc, "## RPMAUTOSPEC:") assert.Contains(t, doc, "leading integer of the committed,") + assert.Contains(t, doc, "committed `-b` value (default `1` when omitted)") + assert.Contains(t, doc, "committed file byte-for-byte") assert.Contains(t, doc, "retains `%autorelease`, `%autochangelog`") assert.Contains(t, doc, "effective upstream distro's `dist-git-branch`") assert.Contains(t, doc, "fetches that historical branch before comparing commits") diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 5d83a9c79..8c4ac008a 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -96,6 +96,11 @@ macro-expanded Release as `-b` and its committed `%changelog` body as the working `changelog` file, removing leading and trailing comment lines starting with `##` while preserving interior comments. Expansion uses the committed files on the host; the later version-change release reset still applies. +Without that legacy marker, an existing `%autorelease` spec carries its +committed `-b` value (default `1` when omitted) into the working macro and +replaces the working `changelog` with the committed file byte-for-byte. +A missing committed `changelog` or invalid numeric base fails rendering. +This carryover requires no host release query; the version-change reset still applies. All prepared dist-git files except `.git` are retained, without file filtering or `spectool` queries. Synthetic history and its legacy release adjustment are skipped during lockless rendering. diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index f6a1876d8..79d28d8e1 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -394,6 +394,7 @@ func locklessRenderRelease( var ( renderedAutoreleaseMarker = regexp.MustCompile(`(?m)^## RPMAUTOSPEC:[^\r\n]*autorelease`) + renderedAutoreleaseBase = regexp.MustCompile(`(?:^|\s)-b(?:\s+|=)?([^\s}]*)`) renderedReleaseNumber = regexp.MustCompile(`^[0-9]+`) ) @@ -401,7 +402,7 @@ func carryRenderedAutorelease( env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, ) (string, error) { if !renderedAutoreleaseMarker.MatchString(history.spec) { - return release, nil + return carryUnprocessedAutorelease(env, dir, release, history) } expanded, err := expandCommittedRelease(env, autospec, dir, filename, history) @@ -432,6 +433,53 @@ func carryRenderedAutorelease( return release, nil } +func carryUnprocessedAutorelease(env *azldev.Env, dir, release string, history lastRender) (string, error) { + previous, err := spec.OpenSpec(strings.NewReader(history.spec)) + if err != nil { + return "", fmt.Errorf("parsing committed autorelease spec:\n%w", err) + } + + previousRelease, err := previous.GetLastTag("", "Release") + if err != nil { + return "", fmt.Errorf("reading committed Release:\n%w", err) + } + + if !sources.ReleaseUsesAutorelease(previousRelease) { + return release, nil + } + + base := "1" + if matches := renderedAutoreleaseBase.FindStringSubmatch(previousRelease); matches != nil { + base = matches[1] + } + + release, err = withAutoreleaseBase(release, base) + if err != nil { + return "", fmt.Errorf("using committed autorelease base:\n%w", err) + } + + if history.tree == nil { + return "", errors.New("missing committed dist-git tree for 'changelog'") + } + + changelog, err := history.tree.File("changelog") + if err != nil { + return "", fmt.Errorf("reading committed 'changelog':\n%w", err) + } + + content, err := changelog.Contents() + if err != nil { + return "", fmt.Errorf("reading committed 'changelog' contents:\n%w", err) + } + + if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "changelog"), []byte(content), + fileperms.PublicFile); err != nil { + return "", fmt.Errorf("writing committed 'changelog':\n%w", err) + } + + return release, nil +} + func stripChangelogBoundaryComments(changelog string) string { lines := strings.SplitAfter(changelog, "\n") for idx, line := range lines { diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index d412c5828..e37c39108 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -660,7 +660,141 @@ func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { } } -func TestLocklessRenderRelease_RequiresAutoreleaseMarker(t *testing.T) { +func TestPrepareLocklessSpec_UnprocessedAutorelease(t *testing.T) { + const changelog = "## Keep committed comments\r\n* Entry\r\n- Keep spaces \r\n\r\n" + + for _, testCase := range []struct { + name, previous, working, want string + upstreamChanged bool + newVersion bool + noWorkingChangelog bool + }{ + { + name: "same version upstream change", previous: "%autorelease -b 17", + working: "%autorelease -b 2 -e test", want: "%autorelease -b 17 -e test", + upstreamChanged: true, + }, + { + name: "braced compact base", previous: "%{autorelease -b17}", + working: "%{autorelease -b 2 -e %{suffix}}", want: "%{autorelease -b 17 -e %{suffix}}", + noWorkingChangelog: true, + }, + { + name: "equals base", previous: "%autorelease -b=17", + working: "%autorelease", want: "%autorelease -b 17", + }, + { + name: "implicit base", previous: "%autorelease", + working: "%autorelease -b 2", want: "%autorelease -b 1", + }, + { + name: "version reset", previous: "%autorelease -b 17", + working: "%autorelease -b 2", want: "%autorelease -b 2", + upstreamChanged: true, newVersion: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + committed := renderTestSpec("1", testCase.previous, "%autochangelog\n") + repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + "curl.spec": []byte(committed), "changelog": []byte(changelog), + }}}) + tree, err := resolveTree(repo, hashes[0]) + require.NoError(t, err) + + env := testutils.NewTestEnv(t) + + version := "1" + if testCase.newVersion { + version = "2" + } + + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(renderTestSpec(version, testCase.working, "%autochangelog\n")), fileperms.PublicFile)) + + if !testCase.noWorkingChangelog { + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", + []byte("replace upstream changelog"), fileperms.PublicFile)) + } + + update := &locklessRenderUpdate{} + if testCase.upstreamChanged { + update.changes = "Upstream fix" + } + + stub := &renderAutospecStub{} + err = prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, + lastRender{spec: committed, tree: tree}, update) + require.NoError(t, err) + assert.Empty(t, stub.calls) + + content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: "+testCase.want+"\n") + assert.Contains(t, string(content), "%changelog\n%autochangelog\n") + content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, changelog, string(content)) + }) + } +} + +func TestPrepareLocklessSpec_UnprocessedAutoreleaseFailure(t *testing.T) { + for _, testCase := range []struct { + name, release, want string + missingFile bool + missingTree bool + }{ + { + name: "missing changelog", release: "%autorelease -b 17", + missingFile: true, want: "reading committed 'changelog'", + }, + { + name: "missing tree", release: "%autorelease -b 17", + missingTree: true, want: "missing committed dist-git tree", + }, + {name: "zero base", release: "%autorelease -b 0", want: "using committed autorelease base"}, + {name: "invalid base", release: "%autorelease -b invalid", want: "using committed autorelease base"}, + } { + t.Run(testCase.name, func(t *testing.T) { + committed := renderTestSpec("1", testCase.release, "%autochangelog\n") + + files := map[string][]byte{"curl.spec": []byte(committed)} + if !testCase.missingFile { + files["changelog"] = []byte("committed changelog") + } + + repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: files}}) + tree, err := resolveTree(repo, hashes[0]) + require.NoError(t, err) + + if testCase.missingTree { + tree = nil + } + + env := testutils.NewTestEnv(t) + original := renderTestSpec("1", "%autorelease -b 2", "%autochangelog\n") + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(original), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", + []byte("upstream changelog"), fileperms.PublicFile)) + + stub := &renderAutospecStub{} + err = prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, + lastRender{spec: committed, tree: tree}, &locklessRenderUpdate{}) + require.ErrorContains(t, err, testCase.want) + assert.Empty(t, stub.calls) + + content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, err) + assert.Equal(t, original, string(content)) + content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, "upstream changelog", string(content)) + }) + } +} + +func TestLocklessRenderRelease_StaticHistoryWithoutAutoreleaseMarker(t *testing.T) { for _, testCase := range []struct{ header, release string }{ {"## RPMAUTOSPEC: autochangelog\n", "%autorelease"}, {"## RPMAUTOSPEC: autochangelog\nautorelease\n", "%autorelease"}, From af484dd52ffe2957a014b472f01e3799a76214f1 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 00:12:24 +0000 Subject: [PATCH 23/70] fix(component): preserve omitted autorelease bases Keep an omitted committed autorelease -b option absent in the working spec instead of adding -b 1. Remove any working numeric base while preserving the macro's other options and committed changelog carryover. Cover bare and braced macro forms and update user and agent guidance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 3 ++- internal/app/azldev/agentskill/agentskill_test.go | 2 +- .../content/refresh-upstream-commit.md.tmpl | 3 ++- .../app/azldev/cmds/component/render_lockless.go | 13 ++++++------- .../cmds/component/render_lockless_internal_test.go | 12 ++++++++++-- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 36a6b28b8..fab67019c 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -158,7 +158,8 @@ The later version-change release reset still applies. For an existing `%autorelease` spec without the legacy marker, rendering carries the committed `-b` value into the working spec, preserving the working macro's -other options. An omitted `-b` means the default base of `1`. The committed +other options. If the committed spec omits `-b`, the working spec also omits it, +removing any working base option rather than adding `-b 1`. The committed `changelog` file replaces the working file byte-for-byte, or is created if absent from the working checkout. A missing committed `changelog` or invalid numeric base fails rendering. No host release or changelog query is needed for this diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 72cf3d998..4e1be06f5 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -192,7 +192,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "automatically commits only that directory") assert.Contains(t, doc, "## RPMAUTOSPEC:") assert.Contains(t, doc, "leading integer of the committed,") - assert.Contains(t, doc, "committed `-b` value (default `1` when omitted)") + assert.Contains(t, doc, "an omitted `-b` stays omitted") assert.Contains(t, doc, "committed file byte-for-byte") assert.Contains(t, doc, "retains `%autorelease`, `%autochangelog`") assert.Contains(t, doc, "effective upstream distro's `dist-git-branch`") diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 8c4ac008a..6dc5bbaee 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -97,7 +97,8 @@ working `changelog` file, removing leading and trailing comment lines starting with `##` while preserving interior comments. Expansion uses the committed files on the host; the later version-change release reset still applies. Without that legacy marker, an existing `%autorelease` spec carries its -committed `-b` value (default `1` when omitted) into the working macro and +committed `-b` value into the working macro; an omitted `-b` stays omitted, +removing any working base option rather than adding `-b 1`. Rendering replaces the working `changelog` with the committed file byte-for-byte. A missing committed `changelog` or invalid numeric base fails rendering. This carryover requires no host release query; the version-change reset still applies. diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 79d28d8e1..09d38b3ec 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -448,14 +448,13 @@ func carryUnprocessedAutorelease(env *azldev.Env, dir, release string, history l return release, nil } - base := "1" if matches := renderedAutoreleaseBase.FindStringSubmatch(previousRelease); matches != nil { - base = matches[1] - } - - release, err = withAutoreleaseBase(release, base) - if err != nil { - return "", fmt.Errorf("using committed autorelease base:\n%w", err) + release, err = withAutoreleaseBase(release, matches[1]) + if err != nil { + return "", fmt.Errorf("using committed autorelease base:\n%w", err) + } + } else { + release = autoreleaseBaseArg.ReplaceAllString(release, "") } if history.tree == nil { diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index e37c39108..8389e70e2 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -684,8 +684,16 @@ func TestPrepareLocklessSpec_UnprocessedAutorelease(t *testing.T) { working: "%autorelease", want: "%autorelease -b 17", }, { - name: "implicit base", previous: "%autorelease", - working: "%autorelease -b 2", want: "%autorelease -b 1", + name: "omitted base", previous: "%autorelease", + working: "%autorelease -b 2 -e test", want: "%autorelease -e test", + }, + { + name: "omitted braced base", previous: "%{autorelease}", + working: "%{autorelease -e %{suffix} -b2}", want: "%{autorelease -e %{suffix}}", + }, + { + name: "both bases omitted", previous: "%autorelease", + working: "%autorelease", want: "%autorelease", }, { name: "version reset", previous: "%autorelease -b 17", From 0008affd4d5237faf8e9ae98004144c6fd977a86 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 00:41:10 +0000 Subject: [PATCH 24/70] fix(component): gate release carryover on version changes Detect version changes immediately after reading the working and committed specs. Calculate or carry release state only for a changed version or missing output, without restoring the original release. Move autorelease changelog generation and carryover into the later changelog phase so they still run when the version is unchanged. Keep upstream change descriptions independent of release preparation. Update regression tests, user documentation and emitted agent guidance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 49 +++--- .../app/azldev/agentskill/agentskill_test.go | 2 + .../content/refresh-upstream-commit.md.tmpl | 37 +++-- internal/app/azldev/cmds/component/render.go | 2 +- .../azldev/cmds/component/render_lockless.go | 145 ++++++++++-------- .../render_lockless_internal_test.go | 132 +++++++++++----- .../component/render_release_carryover.go | 13 +- .../render_release_carryover_internal_test.go | 16 +- 8 files changed, 242 insertions(+), 154 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index fab67019c..b3b1adf39 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -131,39 +131,50 @@ provenance is disabled or the component is not sourced from Fedora upstream. The renderer then finds the existing committed output under `//`, ignoring -uncommitted spec changes. For an existing non-`%autorelease` upstream spec, it +uncommitted spec changes. After reading the working spec's post-overlay +`Release:` and `Version:`, it compares that version with the committed spec's +`Version:`. Release calculation and carryover run **only when the version +differs or no committed output exists**, independently of upstream pin changes. +For an unchanged version, the working release and `baserelease` definition are +left as supplied by upstream and overlays. There is no later restoration of the +original upstream release. + +When release carryover runs for an existing non-`%autorelease` upstream spec, it checks the committed `Release:` value. If it references the `baserelease` macro, the renderer copies the committed `%global baserelease` or `%define baserelease` line over the upstream definition, leaving the upstream `Release:` value unchanged. A missing or ambiguous macro definition fails rendering. Otherwise, it copies the committed `Release:` value into the upstream spec's `Release:`. -No temporary release-bump probe is performed. The committed `%changelog` -section is also preserved. +No temporary release-bump probe is performed. -For an initial `%autorelease` render, it generates the complete upstream -`changelog` file (replacing any existing file), calculates the numeric upstream -release, and adds or replaces the macro's `-b` parameter. These rpmautospec -operations run on the host; they do not preprocess or rewrite the spec. +For an initial `%autorelease` render, it calculates the numeric upstream release +and adds or replaces the macro's `-b` parameter. This rpmautospec operation runs +on the host; it does not preprocess or rewrite the spec. For an existing output whose spec has a line starting with `## RPMAUTOSPEC:` and containing `autorelease` on that same line, a `%autorelease` upstream spec inherits its base from the committed rendered spec. The renderer expands the committed `Release:` with host `rpmspec`, using a temporary copy of the committed dist-git files, and uses its leading integer as `-b` (for example, -`9.fc43` becomes `-b 9`). It also replaces the working checkout's `changelog` -file with the committed `%changelog` body, excluding the section header and -leading or trailing comment lines starting with `##`. Interior comments and -changelog whitespace are preserved. -The later version-change release reset still applies. +`9.fc43` becomes `-b 9`). For an existing `%autorelease` spec without the legacy marker, rendering carries the committed `-b` value into the working spec, preserving the working macro's other options. If the committed spec omits `-b`, the working spec also omits it, -removing any working base option rather than adding `-b 1`. The committed -`changelog` file replaces the working file byte-for-byte, or is created if absent -from the working checkout. A missing committed `changelog` or invalid numeric -base fails rendering. No host release or changelog query is needed for this -carryover; the later version-change release reset still applies. +removing any working base option rather than adding `-b 1`. An invalid numeric +base fails release carryover. No host release query is needed for this carryover. + +Changelog handling follows release preparation and runs even when the version +is unchanged. For non-`%autorelease` specs, the committed inline `%changelog` +section is preserved. For an initial `%autorelease` render, host +`rpmautospec generate-changelog` creates the complete upstream `changelog`, +replacing any existing file. For legacy-marker output, the working `changelog` +is replaced with the committed inline `%changelog` body, excluding the section +header and leading or trailing comment lines starting with `##`; interior +comments and whitespace are preserved. For modern `%autorelease` output, the +committed `changelog` file replaces the working file byte-for-byte, or is created +if absent from the working checkout. A missing committed `changelog` fails +rendering. The last commit touching the component output is the **last render commit**. The renderer fully loads TOML, including includes and component defaults, at @@ -183,10 +194,6 @@ upstream changed. It records `git log --oneline ..` from the upstream checkout; without an old pin it records a one-line version update. An error reading the current pin is fatal. -When upstream changes exist and `Version:` differs from the committed rendered -version (or there is no previous output), it retains the original upstream -`Release:` and `baserelease` definition instead of carrying over previous -values, and restores the upstream `Release:` if an autorelease base was carried over. It then compares the prepared tree with existing output, ignoring the root `.git` and any top-level `.no-change-rebuild` in the existing output only. `--check-only` stops here, before the final release bump. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 4e1be06f5..68c035410 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -194,6 +194,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "leading integer of the committed,") assert.Contains(t, doc, "an omitted `-b` stays omitted") assert.Contains(t, doc, "committed file byte-for-byte") + assert.Contains(t, doc, "Release calculation and carryover run only when the version") + assert.Contains(t, doc, "Changelog handling follows release preparation and runs even when the version") assert.Contains(t, doc, "retains `%autorelease`, `%autochangelog`") assert.Contains(t, doc, "effective upstream distro's `dist-git-branch`") assert.Contains(t, doc, "fetches that historical branch before comparing commits") diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 6dc5bbaee..8086fdf44 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -71,7 +71,7 @@ when writing the macros file; that step never repeats the provenance query. After reading render history and collecting upstream changes, rendering writes the macros file, applies overlays, and updates `sources` before carrying release/changelog state and running the final `rpmdev-bumpspec`. -Carryover and version-reset rules operate on the working spec after overlays. +Version detection and carryover operate on the working spec after overlays. Rendering compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors without replacing existing output. The flag records host `date -Is` output in @@ -79,29 +79,36 @@ without replacing existing output. The flag records host `date -Is` output in ignores this marker in existing output, not in the working upstream checkout. The comparison ignores the root `.git` without traversing or removing it. -For static specs, rendering checks the committed `Release:` value. If it uses +After reading the working spec, rendering compares its `Version:` with the +committed spec. Release calculation and carryover run only when the version +differs or no committed output exists, independently of upstream pin changes. +Otherwise the post-overlay release and `baserelease` definition are unchanged. +There is no later restoration of the original upstream release. + +For static specs, release carryover checks the committed `Release:` value. If it uses the `baserelease` macro, rendering copies the committed `%global baserelease` or `%define baserelease` line over the upstream definition without changing the upstream `Release:` value. Missing or ambiguous definitions fail rendering. Otherwise, it copies the committed `Release:` value. No temporary release-bump -probe is performed, and the committed changelog is retained. A version change -keeps the original upstream `Release:` and `baserelease` definition before -the final host `rpmdev-bumpspec -c` with -`updated ` plus upstream changes. -An initial `%autorelease` render generates the complete upstream -changelog and calculates its base release on the host. For existing output +probe is performed. An initial `%autorelease` render calculates its base release +on the host. For existing output marked by a line starting with `## RPMAUTOSPEC:` and containing `autorelease`, an `%autorelease` upstream spec inherits the leading integer of the committed, -macro-expanded Release as `-b` and its committed `%changelog` body as the -working `changelog` file, removing leading and trailing comment lines starting -with `##` while preserving interior comments. Expansion uses the committed files on the host; -the later version-change release reset still applies. +macro-expanded Release as `-b`. Expansion uses the committed files on the host. Without that legacy marker, an existing `%autorelease` spec carries its committed `-b` value into the working macro; an omitted `-b` stays omitted, -removing any working base option rather than adding `-b 1`. Rendering +removing any working base option rather than adding `-b 1`. +An invalid numeric base fails release carryover, which requires no host release query. + +Changelog handling follows release preparation and runs even when the version +is unchanged. Static specs retain the committed inline `%changelog` section. +An initial `%autorelease` render generates the complete upstream changelog on +the host. Legacy-marker output supplies its committed inline `%changelog` body +as the working `changelog`, removing leading and trailing `##` comment lines +while preserving interior comments and whitespace. Modern autorelease output replaces the working `changelog` with the committed file byte-for-byte. -A missing committed `changelog` or invalid numeric base fails rendering. -This carryover requires no host release query; the version-change reset still applies. +A missing committed `changelog` fails rendering. The final host +`rpmdev-bumpspec -c` still uses `updated ` plus upstream changes. All prepared dist-git files except `.git` are retained, without file filtering or `spectool` queries. Synthetic history and its legacy release adjustment are skipped during lockless rendering. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 2597e8956..0f5dca9dd 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -636,7 +636,7 @@ func finishSourcePreparation( componentName := comp.GetName() if update != nil { if err := prepareLocklessSpec(env, hostRenderProcessor{env: env}, componentDir, componentName, - specEditor, history, update); err != nil { + specEditor, history); err != nil { return nil, fmt.Errorf("preparing release and changelog for %#q:\n%w", componentName, err) } } diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 09d38b3ec..4baec425e 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -272,7 +272,7 @@ func openLocklessSpec( func prepareLocklessSpec( env *azldev.Env, autospec renderAutorelease, dir, name string, editor spec.EditorMode, - history lastRender, update *locklessRenderUpdate, + history lastRender, ) error { path, opened, err := openLocklessSpec(env, dir, name, editor) if err != nil { @@ -289,21 +289,27 @@ func prepareLocklessSpec( return fmt.Errorf("reading upstream Version:\n%w", err) } - release, err := locklessRenderRelease(env, autospec, dir, filepath.Base(path), original, history) + versionChanged, err := locklessVersionChanged(version, history) if err != nil { return err } - release, err = restoreLocklessRelease(original, release, version, history, update.changes != "") - if err != nil { - return err - } + if versionChanged { + release, err := locklessRenderRelease(env, autospec, dir, filepath.Base(path), original, history) + if err != nil { + return err + } - if err := opened.SetTag("", "Release", release); err != nil { - return fmt.Errorf("updating upstream Release:\n%w", err) + if err := opened.SetTag("", "Release", release); err != nil { + return fmt.Errorf("updating upstream Release:\n%w", err) + } + + if err := carryStaticRenderRelease(env, opened, original, history, editor); err != nil { + return err + } } - if err := carryStaticRenderRelease(env, opened, original, version, history, update.changes != "", editor); err != nil { + if err := prepareLocklessChangelog(env, autospec, dir, filepath.Base(path), original, history); err != nil { return err } @@ -374,16 +380,6 @@ func locklessRenderRelease( return carryRenderedAutorelease(env, autospec, dir, filename, release, history) } - changelog, err := autospec.GenerateChangelog(env, dir, filename) - if err != nil { - return "", fmt.Errorf("generating initial upstream changelog:\n%w", err) - } - - if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "changelog"), []byte(changelog), - fileperms.PublicFile); err != nil { - return "", fmt.Errorf("writing initial upstream changelog:\n%w", err) - } - calculated, err := autospec.CalculateBaseRelease(env, dir, filename) if err != nil { return "", fmt.Errorf("calculating upstream release base:\n%w", err) @@ -402,7 +398,7 @@ func carryRenderedAutorelease( env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, ) (string, error) { if !renderedAutoreleaseMarker.MatchString(history.spec) { - return carryUnprocessedAutorelease(env, dir, release, history) + return carryUnprocessedAutorelease(release, history) } expanded, err := expandCommittedRelease(env, autospec, dir, filename, history) @@ -415,25 +411,10 @@ func carryRenderedAutorelease( return "", fmt.Errorf("using committed Release %#q as autorelease base:\n%w", expanded, err) } - previous := []byte(history.spec) - - start, end, err := renderChangelogBounds(previous, spec.EditorLegacy) - if err != nil { - return "", err - } - - _, changelog, _ := strings.Cut(string(previous[start:end]), "\n") - - changelog = stripChangelogBoundaryComments(changelog) - if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "changelog"), []byte(changelog), - fileperms.PublicFile); err != nil { - return "", fmt.Errorf("writing committed 'changelog' content:\n%w", err) - } - return release, nil } -func carryUnprocessedAutorelease(env *azldev.Env, dir, release string, history lastRender) (string, error) { +func carryUnprocessedAutorelease(release string, history lastRender) (string, error) { previous, err := spec.OpenSpec(strings.NewReader(history.spec)) if err != nil { return "", fmt.Errorf("parsing committed autorelease spec:\n%w", err) @@ -457,26 +438,84 @@ func carryUnprocessedAutorelease(env *azldev.Env, dir, release string, history l release = autoreleaseBaseArg.ReplaceAllString(release, "") } + return release, nil +} + +func prepareLocklessChangelog( + env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, +) error { + if !sources.ReleaseUsesAutorelease(release) || env.DryRun() { + return nil + } + + if history.spec != "" && !renderedAutoreleaseMarker.MatchString(history.spec) { + return carryUnprocessedChangelog(env, dir, history) + } + + var changelog string + + if history.spec == "" { + var err error + + changelog, err = autospec.GenerateChangelog(env, dir, filename) + if err != nil { + return fmt.Errorf("generating initial upstream changelog:\n%w", err) + } + } else { + previous := []byte(history.spec) + + start, end, err := renderChangelogBounds(previous, spec.EditorLegacy) + if err != nil { + return err + } + + _, changelog, _ = strings.Cut(string(previous[start:end]), "\n") + changelog = stripChangelogBoundaryComments(changelog) + } + + if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "changelog"), []byte(changelog), + fileperms.PublicFile); err != nil { + return fmt.Errorf("writing prepared 'changelog':\n%w", err) + } + + return nil +} + +func carryUnprocessedChangelog(env *azldev.Env, dir string, history lastRender) error { + previous, err := spec.OpenSpec(strings.NewReader(history.spec)) + if err != nil { + return fmt.Errorf("parsing committed autorelease spec:\n%w", err) + } + + release, err := previous.GetLastTag("", "Release") + if err != nil { + return fmt.Errorf("reading committed Release:\n%w", err) + } + + if !sources.ReleaseUsesAutorelease(release) { + return nil + } + if history.tree == nil { - return "", errors.New("missing committed dist-git tree for 'changelog'") + return errors.New("missing committed dist-git tree for 'changelog'") } changelog, err := history.tree.File("changelog") if err != nil { - return "", fmt.Errorf("reading committed 'changelog':\n%w", err) + return fmt.Errorf("reading committed 'changelog':\n%w", err) } content, err := changelog.Contents() if err != nil { - return "", fmt.Errorf("reading committed 'changelog' contents:\n%w", err) + return fmt.Errorf("reading committed 'changelog' contents:\n%w", err) } if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "changelog"), []byte(content), fileperms.PublicFile); err != nil { - return "", fmt.Errorf("writing committed 'changelog':\n%w", err) + return fmt.Errorf("writing committed 'changelog':\n%w", err) } - return release, nil + return nil } func stripChangelogBoundaryComments(changelog string) string { @@ -552,36 +591,22 @@ func committedRenderSnapshot( return snapshot, nil } -func restoreLocklessRelease( - original, release, version string, history lastRender, upstreamChanged bool, -) (string, error) { - if history.spec == "" && upstreamChanged { - return original, nil - } - +func locklessVersionChanged(version string, history lastRender) (bool, error) { if history.spec == "" { - return release, nil + return true, nil } previous, err := spec.OpenSpec(strings.NewReader(history.spec)) if err != nil { - return "", fmt.Errorf("parsing committed dist-git spec:\n%w", err) - } - - if !upstreamChanged { - return release, nil + return false, fmt.Errorf("parsing committed dist-git spec:\n%w", err) } oldVersion, err := previous.GetLastTag("", "Version") if err != nil { - return "", fmt.Errorf("reading committed Version:\n%w", err) - } - - if oldVersion != version { - return original, nil + return false, fmt.Errorf("reading committed Version:\n%w", err) } - return release, nil + return oldVersion != version, nil } var autoreleaseBaseArg = regexp.MustCompile(`(^|\s)-b(?:\s+|=)?[0-9]+`) diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 8389e70e2..b43c1fe8c 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -428,21 +428,28 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { history: lastRender{ spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "old", }, - wantRelease: "9%{?dist}", + wantRelease: "1%{?dist}", }, { name: "upstream changes without version change", version: "1", release: "1%{?dist}", history: lastRender{ spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "new", }, - wantRelease: "9%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", + wantRelease: "1%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", }, { - name: "version change resets release", version: "2", release: "1%{?dist}", + name: "version change carries release", version: "2", release: "1%{?dist}", history: lastRender{ spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "new", }, - wantRelease: "1%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", + wantRelease: "9%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", + }, + { + name: "version change without upstream changes", version: "2", release: "1%{?dist}", + history: lastRender{ + spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "old", + }, + wantRelease: "9%{?dist}", }, { name: "initial static spec", version: "2", release: "3%{?dist}", @@ -450,7 +457,7 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { }, { name: "initial autorelease", version: "2", release: "%autorelease -b 7", - wantRelease: "%autorelease -b 7", wantChanges: "Update curl to version 2", bootstrap: true, + wantRelease: "%autorelease -b 42", wantChanges: "Update curl to version 2", bootstrap: true, }, { name: "existing autorelease", version: "1", release: "%autorelease", @@ -462,7 +469,7 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { { name: "unknown previous upstream", version: "1", release: "1", history: lastRender{spec: renderTestSpec("1", "9", oldChangelog), current: "new"}, - wantRelease: "9", wantChanges: "Update curl to version 1", + wantRelease: "1", wantChanges: "Update curl to version 1", }, } { t.Run(testCase.name, func(t *testing.T) { @@ -511,7 +518,7 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { require.NoError(t, collectRenderUpstreamChanges(env.Env, "/working", "curl", testCase.version, testCase.history, update)) err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, - testCase.history, update) + testCase.history) require.NoError(t, err) assert.False(t, called, "carryover must not perform the final bump") @@ -521,7 +528,7 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { assert.Equal(t, testCase.wantChanges, update.changes) if testCase.bootstrap { - assert.Equal(t, []string{"changelog", "release"}, stub.calls) + assert.Equal(t, []string{"release", "changelog"}, stub.calls) changelog, err := fileutils.ReadFile(env.TestFS, "/working/changelog") require.NoError(t, err) @@ -586,14 +593,18 @@ func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { upstreamChanged bool wantRelease string }{ - {name: "same upstream", version: "1", wantRelease: "%autorelease -b 9 -e test"}, + {name: "same upstream", version: "1", wantRelease: "%autorelease -b 2 -e test"}, { name: "new upstream same version", version: "1", upstreamChanged: true, + wantRelease: "%autorelease -b 2 -e test", + }, + { + name: "version change", version: "2", upstreamChanged: true, wantRelease: "%autorelease -b 9 -e test", }, { - name: "version reset", version: "2", upstreamChanged: true, - wantRelease: "%autorelease -b 2 -e test", + name: "version change without upstream changes", version: "2", + wantRelease: "%autorelease -b 9 -e test", }, } { t.Run(testCase.name, func(t *testing.T) { @@ -644,18 +655,24 @@ func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { update := &locklessRenderUpdate{subject: "updated curl"} require.NoError(t, collectRenderUpstreamChanges(env.Env, "/working", "curl", testCase.version, history, update)) - err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, history, update) + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, history) require.NoError(t, err) assert.False(t, bumped) _, err = bumpLocklessSpec(env.Env, "/working/curl.spec", update) require.NoError(t, err) assert.True(t, bumped) - assert.Equal(t, []string{"expand"}, stub.calls) - assert.NotEqual(t, "/working", snapshot) - exists, err := fileutils.DirExists(env.TestFS, snapshot) - require.NoError(t, err) - assert.False(t, exists) + + if testCase.version == "1" { + assert.Empty(t, stub.calls) + assert.Empty(t, snapshot) + } else { + assert.Equal(t, []string{"expand"}, stub.calls) + assert.NotEqual(t, "/working", snapshot) + exists, err := fileutils.DirExists(env.TestFS, snapshot) + require.NoError(t, err) + assert.False(t, exists) + } }) } } @@ -665,40 +682,45 @@ func TestPrepareLocklessSpec_UnprocessedAutorelease(t *testing.T) { for _, testCase := range []struct { name, previous, working, want string - upstreamChanged bool newVersion bool noWorkingChangelog bool }{ { - name: "same version upstream change", previous: "%autorelease -b 17", + name: "changed version", previous: "%autorelease -b 17", working: "%autorelease -b 2 -e test", want: "%autorelease -b 17 -e test", - upstreamChanged: true, + newVersion: true, }, { name: "braced compact base", previous: "%{autorelease -b17}", working: "%{autorelease -b 2 -e %{suffix}}", want: "%{autorelease -b 17 -e %{suffix}}", - noWorkingChangelog: true, + noWorkingChangelog: true, newVersion: true, }, { name: "equals base", previous: "%autorelease -b=17", working: "%autorelease", want: "%autorelease -b 17", + newVersion: true, }, { name: "omitted base", previous: "%autorelease", working: "%autorelease -b 2 -e test", want: "%autorelease -e test", + newVersion: true, }, { name: "omitted braced base", previous: "%{autorelease}", working: "%{autorelease -e %{suffix} -b2}", want: "%{autorelease -e %{suffix}}", + newVersion: true, }, { name: "both bases omitted", previous: "%autorelease", working: "%autorelease", want: "%autorelease", }, { - name: "version reset", previous: "%autorelease -b 17", + name: "same version skips release carryover", previous: "%autorelease -b 17", + working: "%autorelease -b 2", want: "%autorelease -b 2", + }, + { + name: "same version skips base removal", previous: "%autorelease", working: "%autorelease -b 2", want: "%autorelease -b 2", - upstreamChanged: true, newVersion: true, }, } { t.Run(testCase.name, func(t *testing.T) { @@ -724,14 +746,9 @@ func TestPrepareLocklessSpec_UnprocessedAutorelease(t *testing.T) { []byte("replace upstream changelog"), fileperms.PublicFile)) } - update := &locklessRenderUpdate{} - if testCase.upstreamChanged { - update.changes = "Upstream fix" - } - stub := &renderAutospecStub{} err = prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, - lastRender{spec: committed, tree: tree}, update) + lastRender{spec: committed, tree: tree}) require.NoError(t, err) assert.Empty(t, stub.calls) @@ -751,6 +768,7 @@ func TestPrepareLocklessSpec_UnprocessedAutoreleaseFailure(t *testing.T) { name, release, want string missingFile bool missingTree bool + newVersion bool }{ { name: "missing changelog", release: "%autorelease -b 17", @@ -760,8 +778,14 @@ func TestPrepareLocklessSpec_UnprocessedAutoreleaseFailure(t *testing.T) { name: "missing tree", release: "%autorelease -b 17", missingTree: true, want: "missing committed dist-git tree", }, - {name: "zero base", release: "%autorelease -b 0", want: "using committed autorelease base"}, - {name: "invalid base", release: "%autorelease -b invalid", want: "using committed autorelease base"}, + { + name: "zero base", release: "%autorelease -b 0", + newVersion: true, want: "using committed autorelease base", + }, + { + name: "invalid base", release: "%autorelease -b invalid", + newVersion: true, want: "using committed autorelease base", + }, } { t.Run(testCase.name, func(t *testing.T) { committed := renderTestSpec("1", testCase.release, "%autochangelog\n") @@ -780,7 +804,13 @@ func TestPrepareLocklessSpec_UnprocessedAutoreleaseFailure(t *testing.T) { } env := testutils.NewTestEnv(t) - original := renderTestSpec("1", "%autorelease -b 2", "%autochangelog\n") + + version := "1" + if testCase.newVersion { + version = "2" + } + + original := renderTestSpec(version, "%autorelease -b 2", "%autochangelog\n") require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", []byte(original), fileperms.PublicFile)) require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", @@ -788,7 +818,7 @@ func TestPrepareLocklessSpec_UnprocessedAutoreleaseFailure(t *testing.T) { stub := &renderAutospecStub{} err = prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, - lastRender{spec: committed, tree: tree}, &locklessRenderUpdate{}) + lastRender{spec: committed, tree: tree}) require.ErrorContains(t, err, testCase.want) assert.Empty(t, stub.calls) @@ -830,7 +860,7 @@ func TestPrepareLocklessSpec_CommittedAutoreleaseFailure(t *testing.T) { } { t.Run(testCase.name, func(t *testing.T) { env := testutils.NewTestEnv(t) - original := renderTestSpec("1", "%autorelease", "%autochangelog\n") + original := renderTestSpec("2", "%autorelease", "%autochangelog\n") require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", []byte(original), fileperms.PublicFile)) require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", @@ -840,7 +870,7 @@ func TestPrepareLocklessSpec_CommittedAutoreleaseFailure(t *testing.T) { return testCase.output, testCase.err }} err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, - testCommittedAutoreleaseHistory(t), &locklessRenderUpdate{subject: "updated curl"}) + testCommittedAutoreleaseHistory(t)) require.ErrorContains(t, err, testCase.want) content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") require.NoError(t, err) @@ -927,6 +957,7 @@ func TestPrepareComponentSources_OverlaysBeforeReleaseCarryover(t *testing.T) { Build: projectconfig.ComponentBuildConfig{With: []string{"feature"}}, Overlays: []projectconfig.ComponentOverlay{ {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Release", Value: "2"}, + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: "2"}, {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Overlaid summary"}, }, SourceFiles: []projectconfig.SourceFileReference{{ @@ -1095,16 +1126,41 @@ func TestPrepareComponentSources_LocklessSkipsSyntheticHistory(t *testing.T) { } } +func TestPrepareLocklessSpec_DetectsVersionBeforeCarryover(t *testing.T) { + env := testutils.NewTestEnv(t) + original := renderTestSpec("2", "%autorelease", "%autochangelog\n") + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(original), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", + []byte("original changelog"), fileperms.PublicFile)) + + history := testCommittedAutoreleaseHistory(t) + history.spec = strings.Replace(history.spec, "Version: 1\n", "", 1) + stub := &renderAutospecStub{expand: func(_, _ string) (string, error) { + return "9", nil + }} + + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, history) + require.ErrorContains(t, err, "reading committed Version") + assert.Empty(t, stub.calls) + + content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, err) + assert.Equal(t, original, string(content)) + content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, "original changelog", string(content)) +} + func TestPrepareLocklessSpec_StopsOnAutospecFailure(t *testing.T) { env := testutils.NewTestEnv(t) require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", []byte(renderTestSpec("1", "%autorelease", "%autochangelog\n")), fileperms.PublicFile)) stub := &renderAutospecStub{err: errors.New("autospec failed")} - err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, lastRender{}, - &locklessRenderUpdate{subject: "updated curl"}) + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, lastRender{}) require.ErrorContains(t, err, "autospec failed") - assert.Equal(t, []string{"changelog"}, stub.calls) + assert.Equal(t, []string{"release"}, stub.calls) } func TestFinishLocklessSpec_TimestampFailures(t *testing.T) { diff --git a/internal/app/azldev/cmds/component/render_release_carryover.go b/internal/app/azldev/cmds/component/render_release_carryover.go index 6cb3890ab..ba257e61c 100644 --- a/internal/app/azldev/cmds/component/render_release_carryover.go +++ b/internal/app/azldev/cmds/component/render_release_carryover.go @@ -19,8 +19,8 @@ var ( ) func carryStaticRenderRelease( - env *azldev.Env, working *spec.Spec, release, version string, - history lastRender, upstreamChanged bool, editor spec.EditorMode, + env *azldev.Env, working *spec.Spec, release string, + history lastRender, editor spec.EditorMode, ) error { if history.spec == "" || sources.ReleaseUsesAutorelease(release) || env.DryRun() { return nil @@ -31,15 +31,6 @@ func carryStaticRenderRelease( return fmt.Errorf("parsing committed release spec:\n%w", err) } - oldVersion, err := previous.GetLastTag("", "Version") - if err != nil { - return fmt.Errorf("reading committed Version:\n%w", err) - } - - if upstreamChanged && oldVersion != version { - return nil - } - oldRelease, err := previous.GetLastTag("", "Release") if err != nil { return fmt.Errorf("reading committed Release:\n%w", err) diff --git a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go index 8d4f4881d..661da93c5 100644 --- a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go +++ b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go @@ -25,24 +25,24 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { }{ { name: "same upstream", version: "1", definition: "%global baserelease 18", - release: "%{baserelease}%{?dist}", wantBase: "18", wantBumped: "19", + release: "%{baserelease}%{?dist}", wantBase: "17", wantBumped: "18", }, { name: "new upstream same version", version: "1", changed: true, definition: "%global baserelease 18", release: "%{baserelease}%{?dist}", - wantBase: "18", wantBumped: "19", + wantBase: "17", wantBumped: "18", }, { name: "new version", version: "2", changed: true, definition: "%global baserelease 18", release: "%{baserelease}%{?dist}", - wantBase: "17", wantBumped: "18", + wantBase: "18", wantBumped: "19", }, { - name: "define and optional macro", version: "1", definition: "%define baserelease 18", + name: "define and optional macro", version: "2", definition: "%define baserelease 18", release: "%{?baserelease}%{?dist}", wantBase: "18", wantBumped: "19", }, { - name: "unbraced macro", version: "1", definition: "%global baserelease 18", + name: "unbraced macro", version: "2", definition: "%global baserelease 18", release: "%baserelease%{?dist}", wantBase: "18", wantBumped: "19", }, } { @@ -82,7 +82,7 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { require.NoError(t, err) definition := testCase.definition - if testCase.changed && testCase.version != "1" { + if testCase.version == "1" { definition = "%define baserelease 17" } @@ -97,7 +97,7 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { update := &locklessRenderUpdate{subject: "updated curl"} require.NoError(t, collectRenderUpstreamChanges(env.Env, "/working", "curl", testCase.version, history, update)) - err := prepareLocklessSpec(env.Env, &renderAutospecStub{}, "/working", "curl", editor, history, update) + err := prepareLocklessSpec(env.Env, &renderAutospecStub{}, "/working", "curl", editor, history) require.NoError(t, err) assert.Zero(t, calls) @@ -158,7 +158,7 @@ func TestCarryStaticRenderRelease(t *testing.T) { history := lastRender{spec: testCase.previous + renderTestSpec("1", testCase.oldRelease, "")} - err = carryStaticRenderRelease(env.Env, opened, "1%{?dist}", "1", history, false, editor) + err = carryStaticRenderRelease(env.Env, opened, "1%{?dist}", history, editor) if testCase.wantError != "" { require.ErrorContains(t, err, testCase.wantError) From 63ba741adb26ba2d45beca4e5944401a7bd758b6 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 01:04:24 +0000 Subject: [PATCH 25/70] fix(component): refine lockless release carryover gates Always calculate initial autorelease bases and carry legacy autorelease bases, regardless of version changes. For modern output, carry the committed base or its omission only when versions match. Otherwise, calculate the upstream numeric base. Omit -b when a calculated base is 1, including legacy expansion, and preserve other macro options. Retain explicit modern bases during unchanged-version carryover. Carry static Release values or baserelease definitions only when committed dist-git output exists and its version matches. Keep working upstream values for new outputs and version changes. Keep changelog handling independent of version changes. Recalculating a modern autorelease base still carries the committed changelog rather than regenerating it. Ignore root .no-change-rebuild markers on both sides when comparing prepared and existing output. Continue comparing nested files with that name. Never replace failed lockless renders with RENDER_FAILED. Leave existing output unchanged until replacement starts. On later failures, clear the component output, reset its staged paths, and check out committed output if the directory previously existed. Otherwise, leave the output absent. Use no recovery backups or index snapshots, and defer lockless aliases until rendering and any automatic commit succeed. Update regression coverage, user documentation, and emitted agent guidance. Validation: mage fix all, mage check all, and mage docs passed. Unit tests passed under mage all; scenarios remain blocked by Docker provider setup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 61 ++-- .../app/azldev/agentskill/agentskill_test.go | 15 +- .../azldev/agentskill/content/azldev.md.tmpl | 6 +- .../content/refresh-upstream-commit.md.tmpl | 23 +- .../content/withoutlockfile/azldev.md.tmpl | 6 +- internal/app/azldev/cmds/component/render.go | 71 ++-- .../cmds/component/render_internal_test.go | 11 +- .../azldev/cmds/component/render_lockless.go | 76 ++++- .../render_lockless_internal_test.go | 167 ++++++--- .../component/render_output_internal_test.go | 323 ++++++++++++++++++ .../render_release_carryover_internal_test.go | 27 +- 11 files changed, 651 insertions(+), 135 deletions(-) create mode 100644 internal/app/azldev/cmds/component/render_output_internal_test.go diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index b3b1adf39..21699f0b6 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -133,11 +133,11 @@ The renderer then finds the existing committed output under `//`, ignoring uncommitted spec changes. After reading the working spec's post-overlay `Release:` and `Version:`, it compares that version with the committed spec's -`Version:`. Release calculation and carryover run **only when the version -differs or no committed output exists**, independently of upstream pin changes. -For an unchanged version, the working release and `baserelease` definition are -left as supplied by upstream and overlays. There is no later restoration of the -original upstream release. +`Version:`, independently of upstream pin changes. Static release carryover runs +**only when committed output exists and the version is unchanged**; otherwise the +working static release and `baserelease` definition are left as supplied by +upstream and overlays. Autorelease follows the case-specific rules below. +There is no later restoration of the original upstream release. When release carryover runs for an existing non-`%autorelease` upstream spec, it checks the committed `Release:` value. If it references the `baserelease` macro, @@ -149,7 +149,8 @@ No temporary release-bump probe is performed. For an initial `%autorelease` render, it calculates the numeric upstream release and adds or replaces the macro's `-b` parameter. This rpmautospec operation runs -on the host; it does not preprocess or rewrite the spec. +on the host; it does not preprocess or rewrite the spec. Initial and legacy +autorelease handling do not depend on version changes. For an existing output whose spec has a line starting with `## RPMAUTOSPEC:` and containing `autorelease` on that same line, a `%autorelease` upstream spec @@ -159,10 +160,19 @@ committed dist-git files, and uses its leading integer as `-b` (for example, `9.fc43` becomes `-b 9`). For an existing `%autorelease` spec without the legacy marker, rendering carries -the committed `-b` value into the working spec, preserving the working macro's -other options. If the committed spec omits `-b`, the working spec also omits it, -removing any working base option rather than adding `-b 1`. An invalid numeric -base fails release carryover. No host release query is needed for this carryover. +the committed `-b` value into the working spec **only when the version is +unchanged**, preserving the working macro's other options. If the committed spec +omits `-b`, the working spec also omits it, removing any working base option +rather than adding `-b 1`. An invalid numeric base fails release carryover. +No host release query is needed for unchanged-version carryover. When the version +changes, rendering calculates the upstream numeric base on the host and adds or +replaces `-b`, just as for an initial render. This changes only release preparation: +the committed changelog is still carried over as described below, not regenerated. + +For calculated bases, including legacy expansion, a base of `1` omits `-b` +and removes any existing working base option, using the `%autorelease` default. +Other macro options are preserved. Unchanged-version modern output still carries +an explicitly committed `-b 1` as-is. Changelog handling follows release preparation and runs even when the version is unchanged. For non-`%autorelease` specs, the committed inline `%changelog` @@ -195,7 +205,7 @@ upstream checkout; without an old pin it records a one-line version update. An error reading the current pin is fatal. It then compares the prepared tree with existing output, ignoring the root `.git` -and any top-level `.no-change-rebuild` in the existing output only. +and any top-level `.no-change-rebuild` on both sides. `--check-only` stops here, before the final release bump. If normal rendering detects no changes, it returns an error indicating there has been no change since the last render, without replacing the existing output @@ -210,8 +220,8 @@ For an unchanged render with this flag, the tool runs host `date -Is` and writes its exact output to `.no-change-rebuild` at the working dist-git root before bumping. The marker is then published and committed with the rendered files. The flag does not create a marker when comparison detects real changes. -A marker already present in the working upstream checkout remains part of the -comparison; only the existing output's marker is ignored. +The root marker is ignored in both the working upstream checkout and existing +output, even if its contents differ. Nested files with the same name are compared. Changed renders, or explicitly requested no-change rebuilds, then invoke host `rpmdev-bumpspec -c` with `updated ` followed by upstream changes on separate lines. @@ -230,22 +240,33 @@ adjustment are skipped. The spec is **not** preprocessed with `rpmautospec proce No mock configuration is needed for lockless rendering. Comparison ignores the root `.git` entry without traversing or removing it; the working checkout retains its Git metadata until staging cleanup. -After comparison and the final release bump, normal rendering -recursively removes the existing component output directory, recreates it, and -moves all prepared files and subdirectories except `.git` into it. Files found -only in the old output are not retained. Moves use filesystem renames, so the -project work directory and rendered output must be on the same filesystem. +After comparison and the final release bump, lockless rendering removes the +existing output directory and moves all prepared entries except `.git` into +its replacement. Files found only in the old output are not retained. +Moves use filesystem renames, so the project work directory and rendered +output must be on the same filesystem. The renderer then commits only the component output directory. The subject is `updated `; the body lists upstream changes, if any. Unrelated staged files are not committed. -Unchanged Git content does not produce an empty commit. A commit failure leaves -the rendered files available for inspection. +Unchanged Git content does not produce an empty commit. +Lockless component failures never create `RENDER_FAILED` or replace existing +output with a failure marker. Failures before output replacement leave the +existing directory unchanged. If replacement or automatic committing fails after +removal begins, rendering clears the component output and resets only that +component's staged paths. If output existed, Git checks out its committed +contents from `HEAD`; otherwise, the directory remains absent. +This recovery does not preserve uncommitted changes or untracked files in the +replaced component directory, and uses no backup or index snapshot. Recovery +errors are reported with the original render failure. +`--check-only` failures leave output untouched and fail the check. +Locked rendering retains its failure marker behavior. For component names requiring URL encoding, rendering also creates a sibling alias such as `libxml%2B%2B -> libxml++` for downstream tooling. Any existing entry at the encoded alias path is removed first, including regular files and nonempty directories, and replaced with a relative symlink. Existing symlink targets are not removed. Alias creation failures produce a warning. +Lockless aliases are updated only after rendering and any automatic commit succeed. `--check-only` does not replace aliases. Install `rpmdevtools`, `rpm-build`, and `rpmautospec` on the host as needed for diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 68c035410..6d7934c0e 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -183,7 +183,9 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") assert.Contains(t, doc, "Unchanged lockless upstream renders require `--no-change-rebuild`") assert.Contains(t, doc, "host `date -Is` output") - assert.Contains(t, doc, "ignores this marker in existing output, not in the working upstream checkout") + assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") + assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") + assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") } doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) @@ -194,7 +196,12 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "leading integer of the committed,") assert.Contains(t, doc, "an omitted `-b` stays omitted") assert.Contains(t, doc, "committed file byte-for-byte") - assert.Contains(t, doc, "Release calculation and carryover run only when the version") + assert.Contains(t, doc, "Static release carryover runs only when committed output exists and the version") + assert.Contains(t, doc, "Initial and legacy autorelease handling do not depend on version changes") + assert.Contains(t, doc, "base only when the version is unchanged") + assert.Contains(t, doc, "When the version changes, rendering calculates the upstream numeric base and sets `-b`") + assert.Contains(t, doc, "The committed changelog is still carried over, not regenerated") + assert.Contains(t, doc, "A calculated base of `1` omits `-b`") assert.Contains(t, doc, "Changelog handling follows release preparation and runs even when the version") assert.Contains(t, doc, "retains `%autorelease`, `%autochangelog`") assert.Contains(t, doc, "effective upstream distro's `dist-git-branch`") @@ -206,7 +213,9 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") assert.Contains(t, doc, "Unchanged lockless upstream renders require `--no-change-rebuild`") assert.Contains(t, doc, "host `date -Is` output") - assert.Contains(t, doc, "ignores this marker in existing output, not in the working upstream checkout") + assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") + assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") + assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 81971ae6d..5c0a82238 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -46,7 +46,11 @@ compares output before the final release bump; `--check-only` stops before that Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors without replacing existing output. The flag records host `date -Is` output in `.no-change-rebuild` before bumping, only when no changes were detected. Comparison -ignores this marker in existing output, not in the working upstream checkout. +ignores this root marker in both existing output and the working upstream checkout. +Lockless failures never write `RENDER_FAILED`. Before replacement, output is +untouched; failures after replacement starts clear the component directory and +use Git to restore committed output, or leave no directory if none existed. +Recovery discards uncommitted changes in that directory and keeps no backups. It never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. It retains all prepared dist-git files except `.git`, without file filtering or `spectool` queries, and replaces and diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 8086fdf44..362cf5114 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -76,13 +76,18 @@ Rendering compares output before the final release bump; `--check-only` stops be Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors without replacing existing output. The flag records host `date -Is` output in `.no-change-rebuild` before bumping, only when no changes were detected. Comparison -ignores this marker in existing output, not in the working upstream checkout. +ignores this root marker in both existing output and the working upstream checkout. +Lockless failures never write `RENDER_FAILED`. Before replacement, output is +untouched; failures after replacement starts clear the component directory and +use Git to restore committed output, or leave no directory if none existed. +Recovery discards uncommitted changes in that directory and keeps no backups. The comparison ignores the root `.git` without traversing or removing it. After reading the working spec, rendering compares its `Version:` with the -committed spec. Release calculation and carryover run only when the version -differs or no committed output exists, independently of upstream pin changes. -Otherwise the post-overlay release and `baserelease` definition are unchanged. +committed spec, independently of upstream pin changes. +Static release carryover runs only when committed output exists and the version +is unchanged. Otherwise the working static release +and `baserelease` definition are unchanged. There is no later restoration of the original upstream release. For static specs, release carryover checks the committed `Release:` value. If it uses @@ -91,14 +96,22 @@ or `%define baserelease` line over the upstream definition without changing the upstream `Release:` value. Missing or ambiguous definitions fail rendering. Otherwise, it copies the committed `Release:` value. No temporary release-bump probe is performed. An initial `%autorelease` render calculates its base release -on the host. For existing output +on the host. Initial and legacy autorelease handling do not depend on version changes. +For existing output marked by a line starting with `## RPMAUTOSPEC:` and containing `autorelease`, an `%autorelease` upstream spec inherits the leading integer of the committed, macro-expanded Release as `-b`. Expansion uses the committed files on the host. Without that legacy marker, an existing `%autorelease` spec carries its +base only when the version is unchanged. For unchanged versions, it carries the committed `-b` value into the working macro; an omitted `-b` stays omitted, removing any working base option rather than adding `-b 1`. An invalid numeric base fails release carryover, which requires no host release query. +When the version changes, rendering calculates the upstream numeric base and sets `-b`, +as for initial output. The committed changelog is still carried over, not regenerated. + +A calculated base of `1` omits `-b`, removing any working base option while preserving +other macro options. This includes legacy expansion. Unchanged-version modern output +still carries an explicitly committed `-b 1` as-is. Changelog handling follows release preparation and runs even when the version is unchanged. Static specs retain the committed inline `%changelog` section. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index a79d561eb..a0c33ac05 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -40,7 +40,11 @@ Rendering compares output before the final release bump; `--check-only` stops be Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors without replacing existing output. The flag records host `date -Is` output in `.no-change-rebuild` before bumping, only when no changes were detected. Comparison -ignores this marker in existing output, not in the working upstream checkout. +ignores this root marker in both existing output and the working upstream checkout. +Lockless failures never write `RENDER_FAILED`. Before replacement, output is +untouched; failures after replacement starts clear the component directory and +use Git to restore committed output, or leave no directory if none existed. +Recovery discards uncommitted changes in that directory and keeps no backups. It replaces and automatically commits only the component output directory. Commit refreshed TOML before rendering; configure Git author identity and install `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 0f5dca9dd..e737327b9 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -169,6 +169,7 @@ type RenderResult struct { Error string `json:"error,omitempty" table:"Error,omitempty"` Changed bool `json:"changed" table:"Changed"` renderErr error `json:"-" table:"-"` + lockless bool `json:"-" table:"-"` } // Render status constants. @@ -268,12 +269,7 @@ func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, parallelFinish(env, prepared, mockResultMap, results, stagingDir, options.Force, options.CheckOnly, options.NoChangeRebuild) - // Write RENDER_FAILED markers for any component that errored in phase 1 - // (source preparation) or phase 3 (mock result application + copy). - // Centralizing this here makes it idempotent with the --clean-stale wipe - // (which sits between phases 2 and 3) and keeps the per-phase code free - // of bookkeeping. In --check-only mode this verifies that on-disk state - // matches the expected single-marker shape and flags drift on mismatch. + // Only locked failures write markers; lockless failures preserve output. writeFailureMarkers(env.FS(), results, options.Force, options.CheckOnly) commitLocklessRenders(env, prepared, results, options.CheckOnly) @@ -413,6 +409,8 @@ type preparedComponent struct { compOutputDir string // validated output path computed in phase 1 update *locklessRenderUpdate lockless bool + outputTouched bool + outputExisted bool } // prepResult pairs a prepared component (on success) or a render result (on error). @@ -480,6 +478,7 @@ func parallelPrepare( OutputDir: compOutputDir, Status: renderStatusCancelled, Error: "context cancelled", + lockless: env.ComponentWithoutLockfile(compName), } case result.Value.result != nil: results[idx] = result.Value.result @@ -519,6 +518,7 @@ func prepareOneComponent( OutputDir: "(invalid)", Status: renderStatusError, Error: nameErr.Error(), + lockless: lockless, }} } @@ -533,6 +533,7 @@ func prepareOneComponent( OutputDir: compOutputDir, Status: renderStatusError, Error: err.Error(), + lockless: lockless, }} } @@ -752,6 +753,7 @@ func parallelFinish( OutputDir: prep.compOutputDir, Status: renderStatusCancelled, Error: "context cancelled", + lockless: prep.lockless, } default: results[prep.index] = result.Value @@ -782,6 +784,7 @@ func finishOneComponent( OutputDir: compOutputDir, Status: renderStatusCancelled, Error: "context cancelled", + lockless: prep.lockless, } } @@ -789,6 +792,7 @@ func finishOneComponent( Component: componentName, OutputDir: compOutputDir, Status: renderStatusOK, + lockless: prep.lockless, } drifted, err := finishComponentRender( @@ -879,7 +883,7 @@ func finishComponentRender( } // Move rendered files to the component's output directory. - if moveErr := moveRenderedOutput(env, componentDir, prep.compOutputDir, allowOverwrite); moveErr != nil { + if moveErr := moveRenderedOutput(env, componentDir, prep, allowOverwrite); moveErr != nil { return changed, moveErr } @@ -892,10 +896,8 @@ func finishComponentRender( // //nolint:godox // tracked by TODO(koji-fragment-decode) tag. // TODO(koji-fragment-decode): remove once the build system decodes fragments. - if aliasErr := writeAliasSymlink(env.FS(), prep.compOutputDir, componentName); aliasErr != nil { - slog.Warn("Failed to create rendered-spec alias symlink; downstream build steps"+ - " that consume the percent-encoded path may fail to locate this component", - "component", componentName, "error", aliasErr) + if !prep.lockless { + writeRenderedAlias(env, prep) } slog.Info("Rendered component", "component", componentName, @@ -904,6 +906,15 @@ func finishComponentRender( return changed, nil } +func writeRenderedAlias(env *azldev.Env, prep *preparedComponent) { + name := prep.comp.GetName() + if err := writeAliasSymlink(env.FS(), prep.compOutputDir, name); err != nil { + slog.Warn("Failed to create rendered-spec alias symlink; downstream build steps"+ + " that consume the percent-encoded path may fail to locate this component", + "component", name, "error", err) + } +} + // writeAliasSymlink creates a sibling symlink alongside componentOutputDir at the // URL-encoded form of componentName, pointing back at the real directory with a // relative target. @@ -945,7 +956,9 @@ func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName st // moveRenderedOutput recreates the output directory and moves all prepared entries // except '.git' into it. Existing output requires allowOverwrite. -func moveRenderedOutput(env *azldev.Env, tempDir, componentOutputDir string, allowOverwrite bool) error { +func moveRenderedOutput(env *azldev.Env, tempDir string, prep *preparedComponent, allowOverwrite bool) error { + componentOutputDir := prep.compOutputDir + if env.DryRun() { slog.Info("Dry run; would replace rendered output", "source", tempDir, "dest", componentOutputDir) @@ -957,13 +970,16 @@ func moveRenderedOutput(env *azldev.Env, tempDir, componentOutputDir string, all return fmt.Errorf("checking output directory %#q:\n%w", componentOutputDir, existsErr) } - if exists { - if !allowOverwrite { - return fmt.Errorf( - "output directory %#q already exists; use --force to overwrite", - componentOutputDir) - } + if exists && !allowOverwrite { + return fmt.Errorf( + "output directory %#q already exists; use --force to overwrite", + componentOutputDir) + } + + prep.outputExisted = exists + prep.outputTouched = true + if exists { // Clean up existing rendered output for this component. if removeErr := env.FS().RemoveAll(componentOutputDir); removeErr != nil { return fmt.Errorf("cleaning output directory %#q:\n%w", componentOutputDir, removeErr) @@ -1163,10 +1179,9 @@ func pruneOrphanRenderedDirs( return nil } -// writeFailureMarkers walks the final results slice and writes a RENDER_FAILED -// marker into each errored component's output directory. When allowOverwrite -// is set, any pre-existing content at the path is removed first so the marker -// isn't surrounded by stale render output. +// writeFailureMarkers writes a RENDER_FAILED marker for errored locked components. +// Lockless failures preserve output, and fail check-only without requiring markers. +// When allowOverwrite is set, pre-existing locked output is removed first. // // Cancelled components are intentionally skipped — a Ctrl-C is not a render // failure, just an incomplete run, and silently planting markers under those @@ -1187,6 +1202,14 @@ func writeFailureMarkers( continue } + if result.lockless { + if checkOnly { + result.Changed = true + } + + continue + } + if checkOnly { drifted, err := outputDriftsFromMarker(fileSystem, result.OutputDir) if err != nil { @@ -1249,7 +1272,7 @@ const renderErrorMarkerContent = "Rendering failed. See azldev logs for details. // A missing actualDir always counts as drift. Symlinks are compared by target // (filesystems without symlink support skip that check; matches production // render behavior). The root '.git' entry is ignored without traversing it. -// The existing output's root '.no-change-rebuild' marker is also ignored. +// The root '.no-change-rebuild' marker is also ignored on both sides. func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string) (bool, error) { actualExists, err := fileutils.DirExists(fileSystem, actualDir) if err != nil { @@ -1262,7 +1285,7 @@ func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string) (boo } result, err := dirdiff.DiffDirs(fileSystem, actualDir, expectedDir, - dirdiff.WithIgnoredPaths(".git"), dirdiff.WithIgnoredOriginalPaths(noChangeRebuildMarker)) + dirdiff.WithIgnoredPaths(".git", noChangeRebuildMarker)) if err != nil { return false, fmt.Errorf("diffing %#q vs %#q:\n%w", actualDir, expectedDir, err) } diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 5d6a68f83..139051e0f 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -113,7 +113,7 @@ func TestMoveRenderedOutput(t *testing.T) { require.NoError(t, fileutils.WriteFile(fs, oldPath, []byte("old output"), fileperms.PublicFile)) } - err := moveRenderedOutput(env, workingDir, outputDir, testCase.allowOverwrite) + err := moveRenderedOutput(env, workingDir, &preparedComponent{compOutputDir: outputDir}, testCase.allowOverwrite) if testCase.wantErr { require.ErrorContains(t, err, "use --force to overwrite") } else { @@ -978,9 +978,10 @@ func TestDiffRenderedOutput_RebuildMarker(t *testing.T) { wantChanged bool }{ {name: "existing marker ignored", existing: noChangeRebuildMarker}, - {name: "working marker compared", working: noChangeRebuildMarker, wantChanged: true}, - {name: "both markers", existing: noChangeRebuildMarker, working: noChangeRebuildMarker, wantChanged: true}, - {name: "nested marker compared", existing: "nested/" + noChangeRebuildMarker, wantChanged: true}, + {name: "working marker ignored", working: noChangeRebuildMarker}, + {name: "different markers ignored", existing: noChangeRebuildMarker, working: noChangeRebuildMarker}, + {name: "nested existing marker compared", existing: "nested/" + noChangeRebuildMarker, wantChanged: true}, + {name: "nested working marker compared", working: "nested/" + noChangeRebuildMarker, wantChanged: true}, } { t.Run(testCase.name, func(t *testing.T) { t.Parallel() @@ -998,7 +999,7 @@ func TestDiffRenderedOutput_RebuildMarker(t *testing.T) { if testCase.working != "" { require.NoError(t, fileutils.WriteFile(testFS, filepath.Join("/working", testCase.working), - []byte("timestamp\n"), fileperms.PublicFile)) + []byte("working timestamp\n"), fileperms.PublicFile)) } changed, err := diffRenderedOutput(testFS, "/working", "/existing") diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 4baec425e..d9316b7bc 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -294,8 +294,8 @@ func prepareLocklessSpec( return err } - if versionChanged { - release, err := locklessRenderRelease(env, autospec, dir, filepath.Base(path), original, history) + if !versionChanged || sources.ReleaseUsesAutorelease(original) { + release, err := locklessRenderRelease(env, autospec, dir, filepath.Base(path), original, history, versionChanged) if err != nil { return err } @@ -371,12 +371,13 @@ func collectRenderUpstreamChanges( func locklessRenderRelease( env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, + versionChanged bool, ) (string, error) { if !sources.ReleaseUsesAutorelease(release) || env.DryRun() { return release, nil } - if history.spec != "" { + if history.spec != "" && (!versionChanged || renderedAutoreleaseMarker.MatchString(history.spec)) { return carryRenderedAutorelease(env, autospec, dir, filename, release, history) } @@ -385,7 +386,7 @@ func locklessRenderRelease( return "", fmt.Errorf("calculating upstream release base:\n%w", err) } - return withAutoreleaseBase(release, calculated) + return withAutoreleaseBase(release, calculated, true) } var ( @@ -406,7 +407,7 @@ func carryRenderedAutorelease( return "", fmt.Errorf("expanding committed Release:\n%w", err) } - release, err = withAutoreleaseBase(release, renderedReleaseNumber.FindString(strings.TrimSpace(expanded))) + release, err = withAutoreleaseBase(release, renderedReleaseNumber.FindString(strings.TrimSpace(expanded)), true) if err != nil { return "", fmt.Errorf("using committed Release %#q as autorelease base:\n%w", expanded, err) } @@ -430,7 +431,7 @@ func carryUnprocessedAutorelease(release string, history lastRender) (string, er } if matches := renderedAutoreleaseBase.FindStringSubmatch(previousRelease); matches != nil { - release, err = withAutoreleaseBase(release, matches[1]) + release, err = withAutoreleaseBase(release, matches[1], false) if err != nil { return "", fmt.Errorf("using committed autorelease base:\n%w", err) } @@ -611,7 +612,7 @@ func locklessVersionChanged(version string, history lastRender) (bool, error) { var autoreleaseBaseArg = regexp.MustCompile(`(^|\s)-b(?:\s+|=)?[0-9]+`) -func withAutoreleaseBase(release, output string) (string, error) { +func withAutoreleaseBase(release, output string, omitDefault bool) (string, error) { fields := strings.Fields(output) if len(fields) == 0 { return "", errors.New("empty autorelease base") @@ -624,6 +625,10 @@ func withAutoreleaseBase(release, output string) (string, error) { return "", fmt.Errorf("invalid autorelease base %#q", output) } + if omitDefault && value == 1 { + return autoreleaseBaseArg.ReplaceAllString(release, ""), nil + } + if autoreleaseBaseArg.MatchString(release) { return autoreleaseBaseArg.ReplaceAllString(release, "${1}-b "+number), nil } @@ -806,18 +811,63 @@ func commitLocklessRenders( return } - // All project index writes are serialized; source preparation and filtering - // remain parallel. A failed commit leaves rendered output intact for recovery. + // Serialize writes to the project index. for _, prep := range prepared { - if prep.update == nil || results[prep.index].Status != renderStatusOK { + result := results[prep.index] + + if prep.update != nil && result.Status == renderStatusOK { + if err := commitLocklessRender(env, prep.update); err != nil { + result.Status = renderStatusError + result.Error = err.Error() + } + } + + if !prep.lockless { continue } - if err := commitLocklessRender(env, prep.update); err != nil { - results[prep.index].Status = renderStatusError - results[prep.index].Error = err.Error() + if result.Status != renderStatusOK { + if prep.outputTouched { + if err := restoreLocklessOutput(env, prep); err != nil { + result.Error += "\n" + err.Error() + } + } + } else { + writeRenderedAlias(env, prep) + } + } +} + +func restoreLocklessOutput(env *azldev.Env, prep *preparedComponent) error { + if err := env.FS().RemoveAll(prep.compOutputDir); err != nil { + return fmt.Errorf("clearing failed rendered output %#q:\n%w", prep.compOutputDir, err) + } + + root := env.ProjectDir() + if prep.update != nil { + root = prep.update.repoRoot + } + + path, err := filepath.Rel(root, prep.compOutputDir) + if err != nil { + return fmt.Errorf("locating rendered output in the project repository:\n%w", err) + } + + path = filepath.ToSlash(path) + ctx := context.WithoutCancel(env) + + // Reset this path first so checkout cannot reuse files staged by the failed render. + if _, err := git.RunInDir(ctx, env, root, "reset", "--quiet", "--", path); err != nil { + return fmt.Errorf("resetting failed rendered output in the index:\n%w", err) + } + + if prep.outputExisted { + if _, err := git.RunInDir(ctx, env, root, "checkout", "HEAD", "--", path); err != nil { + return fmt.Errorf("restoring committed rendered output %#q:\n%w", prep.compOutputDir, err) } } + + return nil } func commitLocklessRender(env *azldev.Env, update *locklessRenderUpdate) error { diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index b43c1fe8c..cd58815c2 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -391,6 +391,7 @@ upstream-commit = "bbbbbbb" type renderAutospecStub struct { calls []string err error + base string expand func(string, string) (string, error) } @@ -403,6 +404,10 @@ func (s *renderAutospecStub) GenerateChangelog(_ context.Context, _, _ string) ( func (s *renderAutospecStub) CalculateBaseRelease(_ context.Context, _, _ string) (string, error) { s.calls = append(s.calls, "release") + if s.base != "" { + return s.base, s.err + } + return "Calculated release number: 42\n", s.err } @@ -422,34 +427,35 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { wantRelease string wantChanges string bootstrap bool + base string }{ { name: "same upstream", version: "1", release: "1%{?dist}", history: lastRender{ spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "old", }, - wantRelease: "1%{?dist}", + wantRelease: "9%{?dist}", }, { name: "upstream changes without version change", version: "1", release: "1%{?dist}", history: lastRender{ spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "new", }, - wantRelease: "1%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", + wantRelease: "9%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", }, { - name: "version change carries release", version: "2", release: "1%{?dist}", + name: "version change keeps upstream release", version: "2", release: "1%{?dist}", history: lastRender{ spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "new", }, - wantRelease: "9%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", + wantRelease: "1%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", }, { name: "version change without upstream changes", version: "2", release: "1%{?dist}", history: lastRender{ spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "old", }, - wantRelease: "9%{?dist}", + wantRelease: "1%{?dist}", }, { name: "initial static spec", version: "2", release: "3%{?dist}", @@ -459,6 +465,11 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { name: "initial autorelease", version: "2", release: "%autorelease -b 7", wantRelease: "%autorelease -b 42", wantChanges: "Update curl to version 2", bootstrap: true, }, + { + name: "initial default base", version: "2", release: "%autorelease -b 7 -e test", + wantRelease: "%autorelease -e test", wantChanges: "Update curl to version 2", + bootstrap: true, base: "Calculated release number: 1\n", + }, { name: "existing autorelease", version: "1", release: "%autorelease", history: lastRender{ @@ -469,7 +480,7 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { { name: "unknown previous upstream", version: "1", release: "1", history: lastRender{spec: renderTestSpec("1", "9", oldChangelog), current: "new"}, - wantRelease: "1", wantChanges: "Update curl to version 1", + wantRelease: "9", wantChanges: "Update curl to version 1", }, } { t.Run(testCase.name, func(t *testing.T) { @@ -513,7 +524,7 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { return nil } - stub := &renderAutospecStub{} + stub := &renderAutospecStub{base: testCase.base} update := &locklessRenderUpdate{subject: "updated curl"} require.NoError(t, collectRenderUpstreamChanges(env.Env, "/working", "curl", testCase.version, testCase.history, update)) @@ -592,19 +603,24 @@ func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { version string upstreamChanged bool wantRelease string + expanded string }{ - {name: "same upstream", version: "1", wantRelease: "%autorelease -b 2 -e test"}, + {name: "same upstream", version: "1", wantRelease: "%autorelease -b 9 -e test", expanded: "9.fc43"}, { name: "new upstream same version", version: "1", upstreamChanged: true, - wantRelease: "%autorelease -b 2 -e test", + wantRelease: "%autorelease -b 9 -e test", expanded: "9.fc43", }, { name: "version change", version: "2", upstreamChanged: true, - wantRelease: "%autorelease -b 9 -e test", + wantRelease: "%autorelease -b 9 -e test", expanded: "9.fc43", }, { name: "version change without upstream changes", version: "2", - wantRelease: "%autorelease -b 9 -e test", + wantRelease: "%autorelease -b 9 -e test", expanded: "9.fc43", + }, + { + name: "default expanded base", version: "2", + wantRelease: "%autorelease -e test", expanded: "1.fc43", }, } { t.Run(testCase.name, func(t *testing.T) { @@ -632,7 +648,7 @@ func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { require.NoError(t, err) assert.Equal(t, "committed macro definitions", string(content)) - return " 9.fc43\n", nil + return " " + testCase.expanded + "\n", nil }} env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") @@ -663,16 +679,11 @@ func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { require.NoError(t, err) assert.True(t, bumped) - if testCase.version == "1" { - assert.Empty(t, stub.calls) - assert.Empty(t, snapshot) - } else { - assert.Equal(t, []string{"expand"}, stub.calls) - assert.NotEqual(t, "/working", snapshot) - exists, err := fileutils.DirExists(env.TestFS, snapshot) - require.NoError(t, err) - assert.False(t, exists) - } + assert.Equal(t, []string{"expand"}, stub.calls) + assert.NotEqual(t, "/working", snapshot) + exists, err := fileutils.DirExists(env.TestFS, snapshot) + require.NoError(t, err) + assert.False(t, exists) }) } } @@ -684,43 +695,66 @@ func TestPrepareLocklessSpec_UnprocessedAutorelease(t *testing.T) { name, previous, working, want string newVersion bool noWorkingChangelog bool + base string }{ { - name: "changed version", previous: "%autorelease -b 17", - working: "%autorelease -b 2 -e test", want: "%autorelease -b 17 -e test", + name: "changed version recalculates base", previous: "%autorelease -b 17", + working: "%autorelease -b 2 -e test", want: "%autorelease -b 42 -e test", newVersion: true, }, + { + name: "changed version omits default base", previous: "%autorelease -b 17", + working: "%autorelease -b 2 -e test", want: "%autorelease -e test", + newVersion: true, base: "Calculated release number: 1\n", + }, + { + name: "same version retains explicit committed default", previous: "%autorelease -b 1", + working: "%autorelease -b 2", want: "%autorelease -b 1", + }, + { + name: "changed version recalculates braced base", previous: "%{autorelease -b17}", + working: "%{autorelease -b 2 -e %{suffix}}", want: "%{autorelease -b 42 -e %{suffix}}", + newVersion: true, noWorkingChangelog: true, + }, { name: "braced compact base", previous: "%{autorelease -b17}", working: "%{autorelease -b 2 -e %{suffix}}", want: "%{autorelease -b 17 -e %{suffix}}", - noWorkingChangelog: true, newVersion: true, + noWorkingChangelog: true, }, { name: "equals base", previous: "%autorelease -b=17", working: "%autorelease", want: "%autorelease -b 17", - newVersion: true, }, { name: "omitted base", previous: "%autorelease", working: "%autorelease -b 2 -e test", want: "%autorelease -e test", - newVersion: true, }, { name: "omitted braced base", previous: "%{autorelease}", working: "%{autorelease -e %{suffix} -b2}", want: "%{autorelease -e %{suffix}}", - newVersion: true, }, { name: "both bases omitted", previous: "%autorelease", working: "%autorelease", want: "%autorelease", }, { - name: "same version skips release carryover", previous: "%autorelease -b 17", - working: "%autorelease -b 2", want: "%autorelease -b 2", + name: "same version carries release base", previous: "%autorelease -b 17", + working: "%autorelease -b 2", want: "%autorelease -b 17", + }, + { + name: "changed version calculates base despite prior omission", previous: "%autorelease", + working: "%autorelease -b 2", want: "%autorelease -b 42", + newVersion: true, + }, + { + name: "changed version adds omitted working base", previous: "%autorelease -b 17", + working: "%autorelease", want: "%autorelease -b 42", + newVersion: true, }, { - name: "same version skips base removal", previous: "%autorelease", - working: "%autorelease -b 2", want: "%autorelease -b 2", + name: "changed version ignores old invalid base", previous: "%autorelease -b invalid", + working: "%autorelease -b 2", want: "%autorelease -b 42", + newVersion: true, }, } { t.Run(testCase.name, func(t *testing.T) { @@ -746,11 +780,16 @@ func TestPrepareLocklessSpec_UnprocessedAutorelease(t *testing.T) { []byte("replace upstream changelog"), fileperms.PublicFile)) } - stub := &renderAutospecStub{} + stub := &renderAutospecStub{base: testCase.base} err = prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, lastRender{spec: committed, tree: tree}) require.NoError(t, err) - assert.Empty(t, stub.calls) + + if testCase.newVersion { + assert.Equal(t, []string{"release"}, stub.calls) + } else { + assert.Empty(t, stub.calls) + } content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") require.NoError(t, err) @@ -769,6 +808,7 @@ func TestPrepareLocklessSpec_UnprocessedAutoreleaseFailure(t *testing.T) { missingFile bool missingTree bool newVersion bool + releaseError error }{ { name: "missing changelog", release: "%autorelease -b 17", @@ -780,11 +820,20 @@ func TestPrepareLocklessSpec_UnprocessedAutoreleaseFailure(t *testing.T) { }, { name: "zero base", release: "%autorelease -b 0", - newVersion: true, want: "using committed autorelease base", + want: "using committed autorelease base", }, { name: "invalid base", release: "%autorelease -b invalid", - newVersion: true, want: "using committed autorelease base", + want: "using committed autorelease base", + }, + { + name: "changed version query failure", release: "%autorelease -b 17", + newVersion: true, releaseError: errors.New("query failed"), + want: "calculating upstream release base", + }, + { + name: "changed version missing changelog", release: "%autorelease -b 17", + newVersion: true, missingFile: true, want: "reading committed 'changelog'", }, } { t.Run(testCase.name, func(t *testing.T) { @@ -816,11 +865,16 @@ func TestPrepareLocklessSpec_UnprocessedAutoreleaseFailure(t *testing.T) { require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", []byte("upstream changelog"), fileperms.PublicFile)) - stub := &renderAutospecStub{} + stub := &renderAutospecStub{err: testCase.releaseError} err = prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, lastRender{spec: committed, tree: tree}) require.ErrorContains(t, err, testCase.want) - assert.Empty(t, stub.calls) + + if testCase.newVersion { + assert.Equal(t, []string{"release"}, stub.calls) + } else { + assert.Empty(t, stub.calls) + } content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") require.NoError(t, err) @@ -842,7 +896,7 @@ func TestLocklessRenderRelease_StaticHistoryWithoutAutoreleaseMarker(t *testing. env := testutils.NewTestEnv(t) stub := &renderAutospecStub{} history := lastRender{spec: testCase.header + renderTestSpec("1", "9", "old\n")} - got, err := locklessRenderRelease(env.Env, stub, "/working", "curl.spec", testCase.release, history) + got, err := locklessRenderRelease(env.Env, stub, "/working", "curl.spec", testCase.release, history, false) require.NoError(t, err) assert.Equal(t, testCase.release, got) assert.Empty(t, stub.calls) @@ -883,29 +937,30 @@ func TestPrepareLocklessSpec_CommittedAutoreleaseFailure(t *testing.T) { } func TestWithAutoreleaseBase(t *testing.T) { - for _, testCase := range []struct{ release, want string }{ - {"%autorelease", "%autorelease -b 42"}, - {"%autorelease -b 7 -e test", "%autorelease -b 42 -e test"}, - {"%autorelease -b7", "%autorelease -b 42"}, - {"%{autorelease}", "%{autorelease -b 42}"}, - {"%{autorelease -b 7}", "%{autorelease -b 42}"}, - {"%autorelease -e %{suffix}", "%autorelease -e %{suffix} -b 42"}, - {"%{autorelease}%{?dist}", "%{autorelease -b 42}%{?dist}"}, - {"%{autorelease -e %{suffix}}", "%{autorelease -e %{suffix} -b 42}"}, + for _, testCase := range []struct{ release, want, wantDefault string }{ + {"%autorelease", "%autorelease -b 42", "%autorelease"}, + {"%autorelease -b 7 -e test", "%autorelease -b 42 -e test", "%autorelease -e test"}, + {"%autorelease -b7", "%autorelease -b 42", "%autorelease"}, + {"%autorelease -b=7 -e test", "%autorelease -b 42 -e test", "%autorelease -e test"}, + {"%{autorelease}", "%{autorelease -b 42}", "%{autorelease}"}, + {"%{autorelease -b 7}", "%{autorelease -b 42}", "%{autorelease}"}, + {"%autorelease -e %{suffix}", "%autorelease -e %{suffix} -b 42", "%autorelease -e %{suffix}"}, + {"%{autorelease}%{?dist}", "%{autorelease -b 42}%{?dist}", "%{autorelease}%{?dist}"}, + {"%{autorelease -e %{suffix}}", "%{autorelease -e %{suffix} -b 42}", "%{autorelease -e %{suffix}}"}, } { - got, err := withAutoreleaseBase(testCase.release, "Calculated release number: 42\n") + got, err := withAutoreleaseBase(testCase.release, "Calculated release number: 42\n", true) require.NoError(t, err) assert.Equal(t, testCase.want, got) + + got, err = withAutoreleaseBase(testCase.release, "Calculated release number: 1\n", true) + require.NoError(t, err) + assert.Equal(t, testCase.wantDefault, got) } for _, output := range []string{"", "release: invalid", "0", "7.fc43"} { - _, err := withAutoreleaseBase("%autorelease", output) + _, err := withAutoreleaseBase("%autorelease", output, true) require.Error(t, err) } - - got, err := withAutoreleaseBase("%autorelease", "1") - require.NoError(t, err) - assert.Equal(t, "%autorelease -b 1", got) } func TestPrepareComponentSources_LocalAndLockedSkipNewLifecycle(t *testing.T) { @@ -957,7 +1012,7 @@ func TestPrepareComponentSources_OverlaysBeforeReleaseCarryover(t *testing.T) { Build: projectconfig.ComponentBuildConfig{With: []string{"feature"}}, Overlays: []projectconfig.ComponentOverlay{ {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Release", Value: "2"}, - {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: "2"}, + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: "1"}, {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Overlaid summary"}, }, SourceFiles: []projectconfig.SourceFileReference{{ @@ -994,7 +1049,7 @@ upstream-commit = "aaaaaaa" manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).DoAndReturn( func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { return fileutils.WriteFile(env.TestFS, filepath.Join(dir, "curl.spec"), - []byte(renderTestSpec("1", "1", "* Upstream entry\n")), fileperms.PublicFile) + []byte(renderTestSpec("2", "1", "* Upstream entry\n")), fileperms.PublicFile) }) originalPreparer := newSourcePreparer diff --git a/internal/app/azldev/cmds/component/render_output_internal_test.go b/internal/app/azldev/cmds/component/render_output_internal_test.go new file mode 100644 index 000000000..e455a51ee --- /dev/null +++ b/internal/app/azldev/cmds/component/render_output_internal_test.go @@ -0,0 +1,323 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "os/exec" + "path/filepath" + "strconv" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/dirdiff" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +const ( + testRenderOutputDir = "/project/SPECS/c/curl" + renderTestDiffCommand = "diff" + renderTestResetCommand = "reset" + renderTestCheckoutCommand = "checkout" + renderTestOverwrite = "overwrite" +) + +func writeOriginalRenderOutput(t *testing.T, fileSystem afero.Fs, root string) { + t.Helper() + + for name, content := range map[string]string{ + "curl.spec": "old spec", "extras/stale.patch": "old sidecar", ".no-change-rebuild": "old timestamp", + } { + require.NoError(t, fileutils.WriteFile(fileSystem, filepath.Join(root, name), + []byte(content), fileperms.PublicFile)) + } +} + +func setupRenderOutput(t *testing.T, fileSystem afero.Fs, existing bool) { + t.Helper() + + for name, content := range map[string]string{ + "curl.spec": "new spec", "sidecar": "new sidecar", ".git/HEAD": "upstream", + } { + require.NoError(t, fileutils.WriteFile(fileSystem, filepath.Join("/staging/curl", name), + []byte(content), fileperms.PublicFile)) + } + + if existing { + writeOriginalRenderOutput(t, fileSystem, testRenderOutputDir) + writeOriginalRenderOutput(t, fileSystem, "/before") + } +} + +func assertOriginalRenderOutput(t *testing.T, fileSystem afero.Fs, existing bool) { + t.Helper() + + exists, err := fileutils.DirExists(fileSystem, testRenderOutputDir) + require.NoError(t, err) + assert.Equal(t, existing, exists) + + if existing { + diff, err := dirdiff.DiffDirs(fileSystem, "/before", testRenderOutputDir) + require.NoError(t, err) + assert.Empty(t, diff.Files) + } +} + +func mockRenderRecovery(t *testing.T, env *testutils.TestEnv) *[]string { + t.Helper() + + var commands []string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + exists, err := fileutils.Exists(env.TestFS, testRenderOutputDir) + require.NoError(t, err) + assert.False(t, exists, "failed output must be cleared before Git recovery") + + command := cmd.Args[3] + commands = append(commands, command) + + switch command { + case renderTestResetCommand: + assert.Equal(t, []string{"git", "-C", "/project", renderTestResetCommand, "--quiet", "--", "SPECS/c/curl"}, cmd.Args) + case renderTestCheckoutCommand: + assert.Equal(t, []string{"git", "-C", "/project", renderTestCheckoutCommand, "HEAD", "--", "SPECS/c/curl"}, cmd.Args) + writeOriginalRenderOutput(t, env.TestFS, testRenderOutputDir) + default: + require.FailNow(t, "unexpected Git command", "%v", cmd.Args) + } + + return "", nil + } + + return &commands +} + +func TestMoveLocklessOutput_PublicationFailure(t *testing.T) { + for _, existing := range []bool{false, true} { + for _, failure := range []string{"processing", renderTestOverwrite, "remove", "move"} { + if (failure == renderTestOverwrite || failure == "remove") && !existing { + continue + } + + t.Run(failure+"/"+strconv.FormatBool(existing), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, existing) + env.TestFS = &renderOutputFaultFS{Fs: env.TestFS, failure: failure} + commands := mockRenderRecovery(t, env) + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + prep := &preparedComponent{ + lockless: true, comp: comp, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, + } + + mockResult := &sources.ComponentMockResult{} + if failure == "processing" { + mockResult.Error = errors.New("injected processing failure") + } + + result := finishOneComponent(env.Env, prep, + map[string]*sources.ComponentMockResult{"curl": mockResult}, + "/staging", failure != renderTestOverwrite, false, false) + require.Error(t, result.renderErr) + commitLocklessRenders(env.Env, []*preparedComponent{prep}, []*RenderResult{result}, false) + writeFailureMarkers(env.TestFS, []*RenderResult{result}, true, false) + assertOriginalRenderOutput(t, env.TestFS, existing) + + switch { + case failure == "processing" || failure == renderTestOverwrite: + assert.False(t, prep.outputTouched) + assert.Empty(t, *commands) + case existing: + assert.Equal(t, []string{renderTestResetCommand, renderTestCheckoutCommand}, *commands) + default: + assert.Equal(t, []string{renderTestResetCommand}, *commands) + } + }) + } + } +} + +func TestCommitLocklessRenders_RestoresFailedOutput(t *testing.T) { + for _, existing := range []bool{false, true} { + for _, failure := range []string{"add", renderTestDiffCommand, "commit", ""} { + t.Run(failure+"/"+strconv.FormatBool(existing), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, existing) + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + prepared := []*preparedComponent{{ + lockless: true, comp: comp, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, + update: &locklessRenderUpdate{ + repoRoot: "/project", path: "SPECS/c/curl", subject: "updated curl", + }, + }} + + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { return nil } + result := finishOneComponent(env.Env, prepared[0], + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, false, false) + require.NoError(t, result.renderErr) + + results := []*RenderResult{result} + commands := mockRenderRecovery(t, env) + recoverCommand := env.CmdFactory.RunAndGetOutputHandler + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[3] == failure { + return "", errors.New("injected Git failure") + } + + if cmd.Args[3] == renderTestDiffCommand { + return "SPECS/c/curl/curl.spec\n", nil + } + + if cmd.Args[3] == renderTestResetCommand || cmd.Args[3] == renderTestCheckoutCommand { + return recoverCommand(cmd) + } + + return "", nil + } + + commitLocklessRenders(env.Env, prepared, results, false) + writeFailureMarkers(env.TestFS, results, true, false) + + if failure != "" { + assert.Equal(t, renderStatusError, results[0].Status) + assert.Contains(t, results[0].Error, "injected Git failure") + assertOriginalRenderOutput(t, env.TestFS, existing) + + if existing { + assert.Equal(t, []string{renderTestResetCommand, renderTestCheckoutCommand}, *commands) + } else { + assert.Equal(t, []string{renderTestResetCommand}, *commands) + } + } else { + assert.Equal(t, renderStatusOK, results[0].Status) + assert.Empty(t, *commands) + + content, err := fileutils.ReadFile(env.TestFS, filepath.Join(testRenderOutputDir, "curl.spec")) + require.NoError(t, err) + assert.Equal(t, "new spec", string(content)) + + for _, name := range []string{".git", "extras", renderErrorMarkerFile} { + exists, err := fileutils.Exists(env.TestFS, filepath.Join(testRenderOutputDir, name)) + require.NoError(t, err) + assert.False(t, exists) + } + } + }) + } + } +} + +func TestCommitLocklessRenders_ReportsRecoveryFailure(t *testing.T) { + for _, failure := range []string{renderTestResetCommand, renderTestCheckoutCommand} { + t.Run(failure, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, true) + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[3] == failure { + return "", errors.New("injected recovery failure") + } + + return "", nil + } + prepared := []*preparedComponent{{ + lockless: true, outputTouched: true, outputExisted: true, compOutputDir: testRenderOutputDir, + }} + results := []*RenderResult{{Status: renderStatusError, Error: "original render failure", lockless: true}} + commitLocklessRenders(env.Env, prepared, results, false) + assert.Contains(t, results[0].Error, "original render failure") + assert.Contains(t, results[0].Error, "injected recovery failure") + assert.Equal(t, renderStatusError, results[0].Status) + }) + } +} + +func TestWriteFailureMarkers_LocklessPreservesOutput(t *testing.T) { + for _, existing := range []bool{false, true} { + for _, checkOnly := range []bool{false, true} { + t.Run(strconv.FormatBool(existing)+"/"+strconv.FormatBool(checkOnly), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, existing) + + results := []*RenderResult{{ + Component: "curl", OutputDir: testRenderOutputDir, Status: renderStatusError, lockless: true, + }} + + writeFailureMarkers(env.TestFS, results, true, checkOnly) + assertOriginalRenderOutput(t, env.TestFS, existing) + assert.Equal(t, checkOnly, results[0].Changed) + }) + } + } +} + +func TestPrepareOneComponent_LocklessFailurePreservesOutput(t *testing.T) { + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, _ opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return nil, errors.New("injected source preparation failure") + } + + t.Cleanup(func() { newSourcePreparer = original }) + + for _, existing := range []bool{false, true} { + t.Run(strconv.FormatBool(existing), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, existing) + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + }) + + prep := prepareOneComponent(env.Env, nil, comp, "/staging", "/project/SPECS", + spec.EditorLegacy, false, true) + require.NotNil(t, prep.result) + assert.Contains(t, prep.result.Error, "injected source preparation failure") + assert.True(t, prep.result.lockless) + writeFailureMarkers(env.TestFS, []*RenderResult{prep.result}, true, false) + assertOriginalRenderOutput(t, env.TestFS, existing) + }) + } +} + +type renderOutputFaultFS struct { + afero.Fs + failure string +} + +func (fs *renderOutputFaultFS) Rename(oldName, newName string) error { + if fs.failure == "move" && oldName == "/staging/curl/sidecar" { + return errors.New("injected rename failure") + } + + return fs.Fs.Rename(oldName, newName) +} + +func (fs *renderOutputFaultFS) RemoveAll(path string) error { + if fs.failure == "remove" && path == testRenderOutputDir { + fs.failure = "" + if err := fs.Fs.RemoveAll(filepath.Join(path, "curl.spec")); err != nil { + return err + } + + return errors.New("injected partial removal failure") + } + + return fs.Fs.RemoveAll(path) +} diff --git a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go index 661da93c5..6becbbdc2 100644 --- a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go +++ b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go @@ -22,29 +22,34 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { for _, testCase := range []struct { name, version, definition, release, wantBase, wantBumped string changed bool + initial bool }{ { name: "same upstream", version: "1", definition: "%global baserelease 18", - release: "%{baserelease}%{?dist}", wantBase: "17", wantBumped: "18", + release: "%{baserelease}%{?dist}", wantBase: "18", wantBumped: "19", }, { name: "new upstream same version", version: "1", changed: true, definition: "%global baserelease 18", release: "%{baserelease}%{?dist}", - wantBase: "17", wantBumped: "18", + wantBase: "18", wantBumped: "19", }, { name: "new version", version: "2", changed: true, definition: "%global baserelease 18", release: "%{baserelease}%{?dist}", - wantBase: "18", wantBumped: "19", + wantBase: "17", wantBumped: "18", }, { - name: "define and optional macro", version: "2", definition: "%define baserelease 18", + name: "define and optional macro", version: "1", definition: "%define baserelease 18", release: "%{?baserelease}%{?dist}", wantBase: "18", wantBumped: "19", }, { - name: "unbraced macro", version: "2", definition: "%global baserelease 18", + name: "unbraced macro", version: "1", definition: "%global baserelease 18", release: "%baserelease%{?dist}", wantBase: "18", wantBumped: "19", }, + { + name: "initial render", version: "1", initial: true, + release: "%{baserelease}%{?dist}", wantBase: "17", wantBumped: "18", + }, } { t.Run(testCase.name, func(t *testing.T) { env := testutils.NewTestEnv(t) @@ -59,6 +64,14 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { renderTestSpec("1", testCase.release, oldChangelog) history := lastRender{spec: oldSpec, previous: "old", current: "old"} + wantChangelog := oldChangelog + + if testCase.initial { + oldSpec = "" + history = lastRender{} + wantChangelog = "* Upstream entry\n" + } + if testCase.changed { history.current = "new" } @@ -82,13 +95,13 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { require.NoError(t, err) definition := testCase.definition - if testCase.version == "1" { + if testCase.wantBase == "17" { definition = "%define baserelease 17" } assert.Contains(t, string(content), definition+"\n") assert.Contains(t, string(content), "Release: "+newRelease+"\n") - assert.Contains(t, string(content), "%changelog\n"+oldChangelog) + assert.Contains(t, string(content), "%changelog\n"+wantChangelog) return fileutils.WriteFile(env.TestFS, path, []byte(strings.Replace(string(content), "baserelease "+testCase.wantBase, From 93ad59da895e4c27a140b3d808af5c56ce65d4fd Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 03:15:43 +0000 Subject: [PATCH 26/70] feat(component): customize lockless render comments and commits Add --changelog-comment for lockless upstream renders. Pass its exact value to rpmdev-bumpspec -c and use it as the automatic Git commit subject. Default to update when the flag is omitted, and retain upstream changes only in the Git commit body. Add --no-git-commit to leave rendered output uncommitted without staging it. Keep the final release bump, publication, failure recovery, and alias handling active. Locked and local component behavior is unchanged. Cover default, custom, explicitly empty, and no-commit cases through the preparation, bump, publication, and automatic commit pipeline. Update user guidance, emitted skills, and generated CLI documentation. Validation: mage fix all, mage check all, mage docs, and mage unit passed. CLI and emitted-skill smoke checks passed. mage all and mage scenarioUpdate remain blocked by unavailable Docker providers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 30 ++- .../reference/cli/azldev_component_render.md | 18 +- .../app/azldev/agentskill/agentskill_test.go | 6 + .../azldev/agentskill/content/azldev.md.tmpl | 6 +- .../content/refresh-upstream-commit.md.tmpl | 9 +- .../content/withoutlockfile/azldev.md.tmpl | 4 + internal/app/azldev/cmds/component/render.go | 53 ++-- .../azldev/cmds/component/render_lockless.go | 20 +- .../render_lockless_internal_test.go | 248 +++++++++++------- .../component/render_output_internal_test.go | 3 +- .../app/azldev/cmds/component/render_test.go | 21 +- 11 files changed, 269 insertions(+), 149 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 21699f0b6..184d2dfe8 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -223,10 +223,16 @@ The flag does not create a marker when comparison detects real changes. The root marker is ignored in both the working upstream checkout and existing output, even if its contents differ. Nested files with the same name are compared. -Changed renders, or explicitly requested no-change rebuilds, then invoke host `rpmdev-bumpspec -c` with -`updated ` followed by upstream changes on separate lines. +Changed renders, or explicitly requested no-change rebuilds, then invoke host +`rpmdev-bumpspec -c` with the exact `--changelog-comment` value. If omitted, the +comment defaults to `update ` for each rendered component. +Upstream changes are not appended to this comment. The tool's no-op for `%autorelease` specs is accepted. +```bash +azldev comp render -p curl --changelog-comment "Fix curl patches" +``` + Macro-file generation, existing overlays, and `sources` manifest updates run before release/changelog carryover and the final `rpmdev-bumpspec` invocation. Carryover and version-reset decisions therefore use the working spec after @@ -246,9 +252,18 @@ its replacement. Files found only in the old output are not retained. Moves use filesystem renames, so the project work directory and rendered output must be on the same filesystem. The renderer then commits only the component output directory. The subject is -`updated `; the body lists upstream changes, if any. Unrelated -staged files are not committed. +the same changelog comment, defaulting to `update `; the body +lists upstream changes, if any. Unrelated staged files are not committed. Unchanged Git content does not produce an empty commit. +Pass `--no-git-commit` to skip automatic staging and committing while still +rendering and bumping the spec. The rendered output remains available for review +and manual commit. Both flags affect only lockless upstream components; +locked and local rendering retain their existing behavior. + +```bash +azldev comp render -p curl --changelog-comment "Fix curl patches" --no-git-commit +``` + Lockless component failures never create `RENDER_FAILED` or replace existing output with a failure marker. Failures before output replacement leave the existing directory unchanged. If replacement or automatic committing fails after @@ -270,9 +285,10 @@ Lockless aliases are updated only after rendering and any automatic commit succe `--check-only` does not replace aliases. Install `rpmdevtools`, `rpm-build`, and `rpmautospec` on the host as needed for -the release/changelog queries, and configure Git author identity before -rendering. Output (including a custom `--output-dir`) must be inside the project -Git repository. `--check-only` prepares and compares temporary output but never +the release/changelog queries. Output (including a custom `--output-dir`) must be inside the project +Git repository. Git author identity is needed for automatic commits; it is not +required by the renderer with `--no-git-commit`. +`--check-only` prepares and compares temporary output but never performs the final `rpmdev-bumpspec` invocation, replaces rendered files, or creates project commits. `--check-only` also never creates a rebuild timestamp, even with `--no-change-rebuild`. The no-change refusal and rebuild flag apply only diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index ace1737a7..5b677b375 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -29,15 +29,16 @@ use lock files. For lockless upstream components, write macros, apply overlays, and update sources before carrying release and changelog state from the last committed render. Compare output; unchanged renders require --no-change-rebuild to record a timestamp. -Then run rpmdev-bumpspec with "updated " and upstream changes. -After replacing output, commit only that directory, excluding unrelated staged files. +Then run rpmdev-bumpspec -c with --changelog-comment (default: "update "). +Commit only that directory with the same subject and upstream changes as the body. +--no-git-commit skips staging and committing. Both flags affect only lockless upstream +components; unrelated staged files are excluded. Output must be inside the project repository; configure Git author identity and install rpmdev-bumpspec. Commit updated upstream-commit TOML before rendering. --check-only stops before bumping, replacing output, or committing. -Lockless rendering runs entirely on the host: it never uses mock or -rpmautospec process-distgit. The rendered spec retains its RPM macros, including -%autorelease and %autochangelog. All prepared dist-git files are retained except -.git; no file filtering or spectool query is performed. Host +Lockless rendering never uses mock or rpmautospec process-distgit. +The spec retains its RPM macros, including %autorelease and %autochangelog. +All prepared files except .git are retained, without filtering or spectool. Host rpmautospec generate-changelog/calculate-release are used only when needed for release history or provenance. Provenance is captured before carryover/bumping and reused for the macros file. Locked components keep their existing behavior. @@ -67,15 +68,13 @@ azldev component render [flags] # Render to a custom directory, allowing removal of existing rendered component directories azldev component render -a -o rendered/ --force - - # Render all and remove stale directories - azldev component render -a --clean-stale ``` ### Options ``` -a, --all-components Include all components + --changelog-comment string comment passed verbatim to rpmdev-bumpspec -c and used as the Git commit subject for lockless upstream components (default: update ) --check-only render to a staging area and compare against the existing on-disk output without modifying the output directory. Exits 0 when nothing would change and 1 when any component would drift. With -a + --clean-stale, also fails on orphan rendered-spec directories. Intended for CI gates. --clean-stale prune rendered-spec directories that no longer correspond to a configured component (only with -a; requires -f with -o). Top-level non-component siblings are preserved. -p, --component stringArray Component name pattern @@ -84,6 +83,7 @@ azldev component render [flags] -f, --force allow overwriting existing rendered component directories -h, --help help for render --no-change-rebuild allow unchanged lockless upstream renders by recording 'date -Is' output in '.no-change-rebuild' + --no-git-commit leave lockless upstream rendered output uncommitted without staging it -o, --output-dir string output directory for rendered specs (overrides rendered-specs-dir from config) --rpmdev-bumpspec Use rpmdev-bumpspec instead of the legacy static release calculation --skip-lock-validation skip lock file consistency checks diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 6d7934c0e..ae7ab9dd5 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -186,6 +186,9 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") + assert.Contains(t, doc, "`--changelog-comment`") + assert.Contains(t, doc, "`update `") + assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") } doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) @@ -216,6 +219,9 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") + assert.Contains(t, doc, "`--changelog-comment`") + assert.Contains(t, doc, "`update `") + assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 5c0a82238..e2f6d225a 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -55,7 +55,11 @@ It never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpande It retains all prepared dist-git files except `.git`, without file filtering or `spectool` queries, and replaces and automatically commits only that component's output directory; unrelated staged -files are excluded. Commit refreshed TOML before rendering. Configure Git author +files are excluded. `--changelog-comment` supplies the exact `rpmdev-bumpspec -c` +value and commit subject, defaulting to `update `. +`--no-git-commit` skips automatic staging and committing, but still renders and bumps. +Upstream changes remain in the Git commit body, not the changelog comment. +Commit refreshed TOML before rendering. Configure Git author identity and install `rpmdevtools`. `--check-only` never creates project commits. Local components skip upstream carryover and automatic commits; locked components retain their existing render behavior. diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 362cf5114..8cfd05af8 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -121,7 +121,8 @@ as the working `changelog`, removing leading and trailing `##` comment lines while preserving interior comments and whitespace. Modern autorelease output replaces the working `changelog` with the committed file byte-for-byte. A missing committed `changelog` fails rendering. The final host -`rpmdev-bumpspec -c` still uses `updated ` plus upstream changes. +`rpmdev-bumpspec -c` uses the exact `--changelog-comment` value, defaulting to +`update `. Upstream changes are not appended to this comment. All prepared dist-git files except `.git` are retained, without file filtering or `spectool` queries. Synthetic history and its legacy release adjustment are skipped during lockless rendering. @@ -135,8 +136,10 @@ committed lock file can supply the old pin if that commit's top-level render still requires a configured pin. Rendering then replaces `{{ .RenderedSpecsDir }}///` and -automatically commits only that directory. The subject is -`updated ` and the body contains upstream changes, if any. +automatically commits only that directory. The subject is the same changelog +comment, defaulting to `update `, and the body contains upstream +changes, if any. +`--no-git-commit` skips automatic staging and committing, but still renders and bumps. It never includes unrelated staged files or creates empty commits. Configure Git author identity and install host `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. `--check-only` never replaces output, creates rebuild timestamps, or commits. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index a0c33ac05..021ce9c11 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -46,6 +46,10 @@ untouched; failures after replacement starts clear the component directory and use Git to restore committed output, or leave no directory if none existed. Recovery discards uncommitted changes in that directory and keeps no backups. It replaces and automatically commits only the component output directory. +`--changelog-comment` supplies the exact `rpmdev-bumpspec -c` value and commit +subject, defaulting to `update `. +`--no-git-commit` skips automatic staging and committing, but still renders and bumps. +Upstream changes remain in the Git commit body, not the changelog comment. Commit refreshed TOML before rendering; configure Git author identity and install `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index e737327b9..4a6f053f2 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -31,15 +31,18 @@ import ( // RenderOptions holds the options for the render command. type RenderOptions struct { componentCommandOptions - ComponentFilter components.ComponentFilter - OutputDir string - OutputDirExplicit bool // True when --output-dir was explicitly passed on the CLI. - FailOnError bool - Force bool - CleanStale bool - CheckOnly bool - NoChangeRebuild bool - RPMDevBumpspec bool + ComponentFilter components.ComponentFilter + OutputDir string + OutputDirExplicit bool // True when --output-dir was explicitly passed on the CLI. + FailOnError bool + Force bool + CleanStale bool + CheckOnly bool + NoChangeRebuild bool + ChangelogComment string + NoGitCommit bool + RPMDevBumpspec bool + changelogCommentExplicit bool } func renderOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { @@ -78,15 +81,16 @@ use lock files. For lockless upstream components, write macros, apply overlays, and update sources before carrying release and changelog state from the last committed render. Compare output; unchanged renders require --no-change-rebuild to record a timestamp. -Then run rpmdev-bumpspec with "updated " and upstream changes. -After replacing output, commit only that directory, excluding unrelated staged files. +Then run rpmdev-bumpspec -c with --changelog-comment (default: "update "). +Commit only that directory with the same subject and upstream changes as the body. +--no-git-commit skips staging and committing. Both flags affect only lockless upstream +components; unrelated staged files are excluded. Output must be inside the project repository; configure Git author identity and install rpmdev-bumpspec. Commit updated upstream-commit TOML before rendering. --check-only stops before bumping, replacing output, or committing. -Lockless rendering runs entirely on the host: it never uses mock or -rpmautospec process-distgit. The rendered spec retains its RPM macros, including -%autorelease and %autochangelog. All prepared dist-git files are retained except -.git; no file filtering or spectool query is performed. Host +Lockless rendering never uses mock or rpmautospec process-distgit. +The spec retains its RPM macros, including %autorelease and %autochangelog. +All prepared files except .git are retained, without filtering or spectool. Host rpmautospec generate-changelog/calculate-release are used only when needed for release history or provenance. Provenance is captured before carryover/bumping and reused for the macros file. Locked components keep their existing behavior. @@ -107,13 +111,11 @@ valid with -a.`, azldev component render -p curl # Render to a custom directory, allowing removal of existing rendered component directories - azldev component render -a -o rendered/ --force - - # Render all and remove stale directories - azldev component render -a --clean-stale`, + azldev component render -a -o rendered/ --force`, RunE: azldev.RunFuncWithExtraArgs(func(env *azldev.Env, args []string) (interface{}, error) { options.ComponentFilter.ComponentNamePatterns = append(args, options.ComponentFilter.ComponentNamePatterns...) options.OutputDirExplicit = cmd.Flags().Changed("output-dir") + options.changelogCommentExplicit = cmd.Flags().Changed("changelog-comment") options.SpecEditor = specEditorFromCommand(cmd) return RenderComponents(env, &options) @@ -152,6 +154,11 @@ func addRenderOptions(cmd *cobra.Command, options *RenderOptions) { "on orphan rendered-spec directories. Intended for CI gates.") cmd.Flags().BoolVar(&options.NoChangeRebuild, "no-change-rebuild", false, "allow unchanged lockless upstream renders by recording 'date -Is' output in '.no-change-rebuild'") + cmd.Flags().StringVar(&options.ChangelogComment, "changelog-comment", "", + "comment passed verbatim to rpmdev-bumpspec -c and used as the Git commit subject "+ + "for lockless upstream components (default: update )") + cmd.Flags().BoolVar(&options.NoGitCommit, "no-git-commit", false, + "leave lockless upstream rendered output uncommitted without staging it") addRPMDevBumpspecFlag(cmd, &options.RPMDevBumpspec) // --check-only is a read-only diff against on-disk state; --fail-on-error @@ -484,6 +491,14 @@ func parallelPrepare( results[idx] = result.Value.result default: result.Value.prepared.index = idx + if update := result.Value.prepared.update; update != nil { + if options.changelogCommentExplicit || options.ChangelogComment != "" { + update.subject = options.ChangelogComment + } + + update.noGitCommit = options.NoGitCommit + } + prepared = append(prepared, result.Value.prepared) } } diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index d9316b7bc..1307e9c60 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -36,10 +36,11 @@ const noChangeRebuildMarker = ".no-change-rebuild" var errNoRenderChanges = errors.New("no change since the last render; use '--no-change-rebuild' to rebuild anyway") type locklessRenderUpdate struct { - repoRoot string - path string - subject string - changes string + repoRoot string + path string + subject string + changes string + noGitCommit bool } type lastRender struct { @@ -82,7 +83,7 @@ func prepareLocklessRender( update = &locklessRenderUpdate{ repoRoot: root, path: path, - subject: "updated " + comp.GetName(), + subject: "update " + comp.GetName(), } _, opened, err := openLocklessSpec(env, dir, comp.GetName(), editor) @@ -775,12 +776,7 @@ func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate return false, fmt.Errorf("reading spec before release bump:\n%w", err) } - entry := update.subject - if update.changes != "" { - entry += "\n" + update.changes - } - - raw := exec.CommandContext(env, sources.RPMDevBumpspecBinary, "-c", entry, path) + raw := exec.CommandContext(env, sources.RPMDevBumpspecBinary, "-c", update.subject, path) raw.Dir = filepath.Dir(path) var stderr bytes.Buffer @@ -815,7 +811,7 @@ func commitLocklessRenders( for _, prep := range prepared { result := results[prep.index] - if prep.update != nil && result.Status == renderStatusOK { + if prep.update != nil && !prep.update.noGitCommit && result.Status == renderStatusOK { if err := commitLocklessRender(env, prep.update); err != nil { result.Status = renderStatusError result.Error = err.Error() diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index cd58815c2..3d362311a 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -506,12 +506,7 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { assert.False(t, called, "only the final release bump should run") called = true - entry := "updated curl" - if testCase.wantChanges != "" { - entry += "\n" + testCase.wantChanges - } - - assert.Equal(t, []string{"rpmdev-bumpspec", "-c", entry, path}, cmd.Args) + assert.Equal(t, []string{"rpmdev-bumpspec", "-c", "update curl", path}, cmd.Args) prepared, err := fileutils.ReadFile(env.TestFS, path) require.NoError(t, err) assert.Contains(t, string(prepared), "Release: "+testCase.wantRelease+"\n") @@ -525,7 +520,7 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { return nil } stub := &renderAutospecStub{base: testCase.base} - update := &locklessRenderUpdate{subject: "updated curl"} + update := &locklessRenderUpdate{subject: "update curl"} require.NoError(t, collectRenderUpstreamChanges(env.Env, "/working", "curl", testCase.version, testCase.history, update)) err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, @@ -1005,28 +1000,44 @@ func TestPrepareComponentSources_LocalAndLockedSkipNewLifecycle(t *testing.T) { } func TestPrepareComponentSources_OverlaysBeforeReleaseCarryover(t *testing.T) { - env := testutils.NewTestEnv(t) - ctrl := gomock.NewController(t) - config := &projectconfig.ComponentConfig{ - Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, - Build: projectconfig.ComponentBuildConfig{With: []string{"feature"}}, - Overlays: []projectconfig.ComponentOverlay{ - {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Release", Value: "2"}, - {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: "1"}, - {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Overlaid summary"}, + for _, testCase := range []struct { + name, comment, wantComment string + explicit, noGitCommit bool + }{ + {name: "default comment", wantComment: "update curl"}, + { + name: "custom comment", comment: "Fix curl's patches; keep $COMPONENT literal", + wantComment: "Fix curl's patches; keep $COMPONENT literal", explicit: true, }, - SourceFiles: []projectconfig.SourceFileReference{{ - Filename: "extra-source.tar.gz", Hash: "abc123", HashType: fileutils.HashTypeSHA512, - Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://example.com/extra-source.tar.gz"}, - }}, - } - comp := components_testutils.NewMockComponent(ctrl) - comp.EXPECT().GetName().AnyTimes().Return("curl") - comp.EXPECT().GetConfig().AnyTimes().Return(config) + {name: "empty explicit comment", explicit: true}, + { + name: "no Git commit", comment: "Prepare curl for review", + wantComment: "Prepare curl for review", noGitCommit: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + ctrl := gomock.NewController(t) + config := &projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + Build: projectconfig.ComponentBuildConfig{With: []string{"feature"}}, + Overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Release", Value: "2"}, + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: "1"}, + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Overlaid summary"}, + }, + SourceFiles: []projectconfig.SourceFileReference{{ + Filename: "extra-source.tar.gz", Hash: "abc123", HashType: fileutils.HashTypeSHA512, + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://example.com/extra-source.tar.gz"}, + }}, + } + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(config) - const ( - oldChangelog = "* Downstream entry\n" - rootConfig = `[project] + const ( + oldChangelog = "* Downstream entry\n" + rootConfig = `[project] default-distro = { name = "test-distro", version = "1.0" } [distros.test-distro.versions."1.0"] release-ver = "3" @@ -1035,85 +1046,130 @@ dist-git-branch = "upstream" type = "upstream" upstream-commit = "aaaaaaa" ` - ) + ) - repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ - "azldev.toml": []byte(rootConfig), - "SPECS/c/curl/curl.spec": []byte(renderTestSpec("1", "9", oldChangelog)), - }}}) - originalOpen := openRenderRepo - openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { - return repo, "/project", nil - } - manager := sourceproviders_test.NewMockSourceManager(ctrl) - manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).DoAndReturn( - func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { - return fileutils.WriteFile(env.TestFS, filepath.Join(dir, "curl.spec"), - []byte(renderTestSpec("2", "1", "* Upstream entry\n")), fileperms.PublicFile) - }) + repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + "azldev.toml": []byte(rootConfig), + "SPECS/c/curl/curl.spec": []byte(renderTestSpec("1", "9", oldChangelog)), + }}}) + originalOpen := openRenderRepo + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { + return repo, "/project", nil + } + manager := sourceproviders_test.NewMockSourceManager(ctrl) + manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile(env.TestFS, filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("2", "1", "* Upstream entry\n")), fileperms.PublicFile) + }) - originalPreparer := newSourcePreparer - newSourcePreparer = func( - _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, - options ...sources.PreparerOption, - ) (sources.SourcePreparer, error) { - return sources.NewPreparer(manager, fs, events, dry, options...) - } + originalPreparer := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } - t.Cleanup(func() { - openRenderRepo = originalOpen - newSourcePreparer = originalPreparer - }) - env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + t.Cleanup(func() { + openRenderRepo = originalOpen + newSourcePreparer = originalPreparer + }) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") - bumped := false - env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { - assert.Equal(t, []string{ - "git", "-C", "/project", "--literal-pathspecs", "log", "-1", "--format=%H", - "--end-of-options", hashes[0], "--", "SPECS/c/curl/", - }, cmd.Args) + bumped := false + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{ + "git", "-C", "/project", "--literal-pathspecs", "log", "-1", "--format=%H", + "--end-of-options", hashes[0], "--", "SPECS/c/curl/", + }, cmd.Args) - return hashes[0], nil - } - env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { - bumped = true + return hashes[0], nil + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + bumped = true - assert.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) - path := cmd.Args[len(cmd.Args)-1] - content, err := fileutils.ReadFile(env.TestFS, path) - require.NoError(t, err) - assert.Contains(t, string(content), "Summary: Overlaid summary\n") - assert.Contains(t, string(content), "Release: 9\n") - assert.Contains(t, string(content), "%changelog\n"+oldChangelog) - assert.Contains(t, string(content), "%{load:%{_sourcedir}/curl.azl.macros}") - assert.Contains(t, string(content), "Source9999") + assert.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) + assert.Equal(t, "-c", cmd.Args[1]) + assert.Equal(t, testCase.wantComment, cmd.Args[2]) + path := cmd.Args[len(cmd.Args)-1] + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Contains(t, string(content), "Summary: Overlaid summary\n") + assert.Contains(t, string(content), "Release: 9\n") + assert.Contains(t, string(content), "%changelog\n"+oldChangelog) + assert.Contains(t, string(content), "%{load:%{_sourcedir}/curl.azl.macros}") + assert.Contains(t, string(content), "Source9999") - macros, err := fileutils.ReadFile(env.TestFS, "/staging/curl/curl.azl.macros") - require.NoError(t, err) - assert.Contains(t, string(macros), "%_with_feature 1\n") + macros, err := fileutils.ReadFile(env.TestFS, "/staging/curl/curl.azl.macros") + require.NoError(t, err) + assert.Contains(t, string(macros), "%_with_feature 1\n") - manifest, err := fileutils.ReadFile(env.TestFS, "/staging/curl/sources") - require.NoError(t, err) - assert.Contains(t, string(manifest), "SHA512 (extra-source.tar.gz) = abc123") + manifest, err := fileutils.ReadFile(env.TestFS, "/staging/curl/sources") + require.NoError(t, err) + assert.Contains(t, string(manifest), "SHA512 (extra-source.tar.gz) = abc123") - return fileutils.WriteFile(env.TestFS, path, - []byte(strings.Replace(string(content), "Release: 9\n", "Release: 10\n", 1)), fileperms.PublicFile) - } - prep, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/project/SPECS/c/curl", - spec.EditorLegacy, false, true) - require.NoError(t, err) - assert.False(t, bumped) + return fileutils.WriteFile(env.TestFS, path, + []byte(strings.Replace(string(content), "Release: 9\n", "Release: 10\n", 1)), fileperms.PublicFile) + } + results := make([]*RenderResult, 1) + prepared := parallelPrepare(env.Env, nil, []components.Component{comp}, "/staging", &RenderOptions{ + OutputDir: "/project/SPECS", ChangelogComment: testCase.comment, + changelogCommentExplicit: testCase.explicit, NoGitCommit: testCase.noGitCommit, + }, results) + require.Len(t, prepared, 1) + prep := prepared[0] + prep.update.changes = "abc123 Fix upstream" - prep.compOutputDir = "/project/SPECS/c/curl" - changed, err := finishComponentRender(env.Env, prep, - map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, false, false) - require.NoError(t, err) - assert.True(t, changed) - assert.True(t, bumped) + assert.False(t, bumped) - content, err := fileutils.ReadFile(env.TestFS, "/project/SPECS/c/curl/curl.spec") - require.NoError(t, err) - assert.Contains(t, string(content), "Release: 10\n") + prep.compOutputDir = "/project/SPECS/c/curl" + changed, err := finishComponentRender(env.Env, prep, + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, false, false) + require.NoError(t, err) + assert.True(t, changed) + assert.True(t, bumped) + + content, err := fileutils.ReadFile(env.TestFS, "/project/SPECS/c/curl/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: 10\n") + + var gitCommands []string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + gitCommands = append(gitCommands, cmd.Args[3]) + + assert.False(t, testCase.noGitCommit, "no Git commands should run for --no-git-commit") + + if cmd.Args[3] == renderTestDiffCommand { + return renderTestSpecDiff, nil + } + + if cmd.Args[3] == "commit" { + assert.Equal(t, []string{ + "git", "-C", "/project", "commit", "--only", "-m", testCase.wantComment, + "-m", "abc123 Fix upstream", "--", "SPECS/c/curl", + }, cmd.Args) + } + + return "", nil + } + results[0] = &RenderResult{Status: renderStatusOK, lockless: true} + commitLocklessRenders(env.Env, prepared, results, false) + + if testCase.noGitCommit { + assert.Empty(t, gitCommands) + } else { + assert.Equal(t, []string{"add", renderTestDiffCommand, "commit"}, gitCommands) + } + + assert.Equal(t, renderStatusOK, results[0].Status) + + content, err = fileutils.ReadFile(env.TestFS, "/project/SPECS/c/curl/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: 10\n") + }) + } } func TestPrepareComponentSources_LocklessSkipsSyntheticHistory(t *testing.T) { @@ -1411,7 +1467,7 @@ func TestCommitLocklessRender_PathOnlyAndNoEmptyCommits(t *testing.T) { env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { commands = append(commands, cmd.Args) if cmd.Args[3] == "diff" && changed { - return "SPECS/c/curl/curl.spec\n", nil + return renderTestSpecDiff, nil } return "", nil diff --git a/internal/app/azldev/cmds/component/render_output_internal_test.go b/internal/app/azldev/cmds/component/render_output_internal_test.go index e455a51ee..c5c774d9c 100644 --- a/internal/app/azldev/cmds/component/render_output_internal_test.go +++ b/internal/app/azldev/cmds/component/render_output_internal_test.go @@ -32,6 +32,7 @@ const ( renderTestResetCommand = "reset" renderTestCheckoutCommand = "checkout" renderTestOverwrite = "overwrite" + renderTestSpecDiff = "SPECS/c/curl/curl.spec\n" ) func writeOriginalRenderOutput(t *testing.T, fileSystem afero.Fs, root string) { @@ -179,7 +180,7 @@ func TestCommitLocklessRenders_RestoresFailedOutput(t *testing.T) { } if cmd.Args[3] == renderTestDiffCommand { - return "SPECS/c/curl/curl.spec\n", nil + return renderTestSpecDiff, nil } if cmd.Args[3] == renderTestResetCommand || cmd.Args[3] == renderTestCheckoutCommand { diff --git a/internal/app/azldev/cmds/component/render_test.go b/internal/app/azldev/cmds/component/render_test.go index aadae63d1..2367053be 100644 --- a/internal/app/azldev/cmds/component/render_test.go +++ b/internal/app/azldev/cmds/component/render_test.go @@ -54,6 +54,15 @@ func TestNewRenderCmd_Flags(t *testing.T) { require.NoError(t, cmd.ParseFlags([]string{"--no-change-rebuild"})) assert.Equal(t, "true", noChangeRebuildFlag.Value.String()) + commentFlag := cmd.Flags().Lookup("changelog-comment") + require.NotNil(t, commentFlag) + + noGitCommitFlag := cmd.Flags().Lookup("no-git-commit") + require.NotNil(t, noGitCommitFlag) + require.NoError(t, cmd.ParseFlags([]string{"--changelog-comment", "Fix curl patches", "--no-git-commit"})) + assert.Equal(t, "Fix curl patches", commentFlag.Value.String()) + assert.Equal(t, "true", noGitCommitFlag.Value.String()) + rpmDevBumpspecFlag := cmd.Flags().Lookup("rpmdev-bumpspec") require.NotNil(t, rpmDevBumpspecFlag) assert.Equal(t, "false", rpmDevBumpspecFlag.DefValue) @@ -66,7 +75,10 @@ func TestRenderCmd_NoComponents(t *testing.T) { testEnv := testutils.NewTestEnv(t) cmd := componentcmds.NewRenderCmd() - cmd.SetArgs([]string{"-o", "SPECS", "--no-change-rebuild", "nonexistent-component"}) + cmd.SetArgs([]string{ + "-o", "SPECS", "--no-change-rebuild", "--no-git-commit", + "--changelog-comment", "Fix patches", "nonexistent-component", + }) err := cmd.ExecuteContext(testEnv.Env) @@ -75,6 +87,13 @@ func TestRenderCmd_NoComponents(t *testing.T) { assert.Contains(t, err.Error(), "component not found") } +func TestRenderCmd_ChangelogCommentRequiresValue(t *testing.T) { + testEnv := testutils.NewTestEnv(t) + cmd := componentcmds.NewRenderCmd() + cmd.SetArgs([]string{"--changelog-comment"}) + require.ErrorContains(t, cmd.ExecuteContext(testEnv.Env), "flag needs an argument") +} + func TestRenderCmd_NoOutputDir(t *testing.T) { testEnv := testutils.NewTestEnv(t) From edd13315e7bfb992cb399cc5aafed380178a3707 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 03:22:23 +0000 Subject: [PATCH 27/70] feat(component): add make-lockless migration command Add azldev comp make-lockless with the standard component selectors. Resolve the selection without lock validation, append missing exact names to the top-level lockless-components.txt, and run the existing refresh-upstream-commit pipeline for that selection. Preserve existing list contents and lock files. Refresh already-listed components without duplicating entries, and update the in-memory mode selection before refreshing. Non-upstream components use the existing generated-pin cleanup behavior. Keep the selection and successful pin updates if refreshing fails so the command can be retried. Dry runs leave both selection and pins untouched. Cover selection, idempotency, missing newlines, refresh failures, local components, and dry runs. Update guides, emitted skills, and CLI docs. Validation: mage fix all, mage check all, mage docs, and mage unit passed. CLI and emitted-skill install/show smoke checks passed. mage all and mage scenarioUpdate remain blocked by unavailable Docker providers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 16 ++ docs/user/reference/cli/azldev_component.md | 1 + .../cli/azldev_component_make-lockless.md | 69 +++++++ .../app/azldev/agentskill/agentskill_test.go | 2 + .../azldev/agentskill/content/azldev.md.tmpl | 3 + .../content/refresh-upstream-commit.md.tmpl | 4 + .../content/withoutlockfile/azldev.md.tmpl | 5 + .../app/azldev/cmds/component/component.go | 1 + .../app/azldev/cmds/component/makelockless.go | 144 ++++++++++++++ .../cmds/component/makelockless_test.go | 182 ++++++++++++++++++ 10 files changed, 427 insertions(+) create mode 100644 docs/user/reference/cli/azldev_component_make-lockless.md create mode 100644 internal/app/azldev/cmds/component/makelockless.go create mode 100644 internal/app/azldev/cmds/component/makelockless_test.go diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 184d2dfe8..3c3113a1f 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -37,6 +37,22 @@ A missing or empty file means all components use lock files. Blank lines and surrounding whitespace are ignored; names are not glob patterns. A file read error stops configuration loading. +To add names and refresh their upstream commits in one command, first configure +the generated TOML include described below, then run: + +```bash +azldev comp make-lockless curl rust-asyncband +``` + +`make-lockless` accepts the normal component selectors (`-p`, `-g`, and `-a`). +It appends missing exact names to the top-level list without rewriting existing +entries or removing lock files, then runs the `refresh-upstream-commit` pipeline +for the selection. Already-listed components are refreshed again without adding +duplicates. Non-upstream components are listed too; refresh removes any generated +pin for them without contacting an upstream provider. +If refreshing fails, the list stays updated and successful pins are saved. +Fix the failure and rerun the command. Dry runs change neither the list nor pins. + Both `component update` and `component refresh-upstream-commit` are available. Update (including `--bump`) skips listed components and leaves their locks untouched; refresh skips unlisted components and preserves their diff --git a/docs/user/reference/cli/azldev_component.md b/docs/user/reference/cli/azldev_component.md index 0ef76adda..9fb435e1b 100644 --- a/docs/user/reference/cli/azldev_component.md +++ b/docs/user/reference/cli/azldev_component.md @@ -46,6 +46,7 @@ components defined in the project configuration. * [azldev component diff-sources](azldev_component_diff-sources.md) - Show the diff that overlays apply to a component's sources * [azldev component history](azldev_component_history.md) - Report per-component change activity and customization detail * [azldev component list](azldev_component_list.md) - List components in this project +* [azldev component make-lockless](azldev_component_make-lockless.md) - Select the lockless workflow and refresh upstream commits * [azldev component metadata](azldev_component_metadata.md) - List documentation metadata for components' overlays and groups * [azldev component prepare-sources](azldev_component_prepare-sources.md) - Prepare buildable sources for components * [azldev component query](azldev_component_query.md) - Query info from locally rendered component specs diff --git a/docs/user/reference/cli/azldev_component_make-lockless.md b/docs/user/reference/cli/azldev_component_make-lockless.md new file mode 100644 index 000000000..73c2df064 --- /dev/null +++ b/docs/user/reference/cli/azldev_component_make-lockless.md @@ -0,0 +1,69 @@ + + +## azldev component make-lockless + +Select the lockless workflow and refresh upstream commits + +### Synopsis + +Add selected component names to lockless-components.txt beside the top-level +azldev.toml, then run the refresh-upstream-commit pipeline for those components. + +Existing entries, whitespace, and ordering are preserved. Missing names are +appended once, using exact component names rather than selection patterns. +Existing lock files are left untouched. Already-listed components are refreshed +again without duplicating their entries. + +Upstream components get generated commit TOML under base/upstream-commits/. +Include that directory in azldev.toml before component-specific configuration. +Non-upstream components are listed too, but do not contact upstream providers. +The refresh pipeline removes their generated commit TOML if one exists. + +If refreshing fails, the updated selection remains in place and successful +component pins are still saved. Fix the failure and rerun this command. + +``` +azldev component make-lockless [flags] +``` + +### Examples + +``` + azldev comp make-lockless -p curl + azldev comp make-lockless curl rust-asyncband + azldev comp make-lockless -g core + azldev comp make-lockless -a +``` + +### Options + +``` + -a, --all-components Include all components + -p, --component stringArray Component name pattern + -g, --component-group stringArray Component group name + -h, --help help for make-lockless + -s, --spec-path stringArray Spec path +``` + +### Options inherited from parent commands + +``` + -y, --accept-all accept all prompts + --color mode output colorization mode {always, auto, never} (default auto) + --concurrency int base concurrency limit (default: number of logical CPUs; minimum 1) + --config-file stringArray additional TOML config file(s) to merge (may be repeated) + -n, --dry-run dry run only (do not take action) + --network-retries int maximum number of attempts for network operations (minimum 1) (default 3) + --no-default-config disable default configuration + -O, --output-format fmt output format {csv, json, markdown, table} (default table) + --permissive-config do not fail on unknown fields in TOML config files + -C, --project string path to Azure Linux project + -q, --quiet only enable minimal output + --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) + -v, --verbose enable verbose output +``` + +### SEE ALSO + +* [azldev component](azldev_component.md) - Manage components + diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index ae7ab9dd5..4fbf246ee 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -189,6 +189,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "`--changelog-comment`") assert.Contains(t, doc, "`update `") assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") + assert.Contains(t, doc, "`azldev comp make-lockless`") } doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) @@ -222,6 +223,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "`--changelog-comment`") assert.Contains(t, doc, "`update `") assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") + assert.Contains(t, doc, "`azldev comp make-lockless`") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index e2f6d225a..2ab67f373 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -37,6 +37,9 @@ images assemble RPMs into bootable artifacts. `lockless-components.txt` beside the top-level `azldev.toml` selects the lockless workflow per component for all commands. Listed components ignore locks and use generated upstream-commit TOML instead. All others use locks. +`azldev comp make-lockless` adds selected exact names to that list if needed, +then runs the refresh pipeline. Existing list contents and lock files are retained. +Already-listed components are still refreshed; a refresh failure leaves the list updated. For listed upstream components, rendering writes macros, applies overlays, and updates `sources` before carrying release/changelog state from the committed diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 8cfd05af8..877517446 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -42,6 +42,10 @@ Add `-O json` for machine-readable output when debugging. Select this workflow by listing the component's exact name in `lockless-components.txt` beside the top-level `azldev.toml`. +Alternatively, `azldev comp make-lockless` adds selected exact names if needed +and runs this refresh pipeline. It accepts `-p`, `-g`, `-a`, or positional names, +preserves existing list contents and lock files, and refreshes already-listed components. +A refresh failure leaves the updated list in place; fix the failure and rerun. Refresh skips unlisted components and preserves their generated TOML files. Commit component inputs before rendering. For upstream components, rendering diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 021ce9c11..8f2d0853a 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -33,6 +33,11 @@ images assemble RPMs into bootable artifacts. ## Common commands +`azldev comp make-lockless` adds selected exact names to the top-level +`lockless-components.txt` if needed, then runs the refresh pipeline. +Existing list contents and lock files are retained. Already-listed components +are still refreshed; a refresh failure leaves the list updated. + For upstream components, rendering writes macros, applies overlays, and updates `sources` before carrying release/changelog state from the committed output and running host `rpmdev-bumpspec`. diff --git a/internal/app/azldev/cmds/component/component.go b/internal/app/azldev/cmds/component/component.go index dced09151..549700739 100644 --- a/internal/app/azldev/cmds/component/component.go +++ b/internal/app/azldev/cmds/component/component.go @@ -41,6 +41,7 @@ components defined in the project configuration.`, queryOnAppInit(app, cmd) updateOnAppInit(app, cmd) refreshUpstreamCommitOnAppInit(app, cmd) + makeLocklessOnAppInit(app, cmd) return cmd } diff --git a/internal/app/azldev/cmds/component/makelockless.go b/internal/app/azldev/cmds/component/makelockless.go new file mode 100644 index 000000000..c89b2dabb --- /dev/null +++ b/internal/app/azldev/cmds/component/makelockless.go @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/cobra" +) + +func makeLocklessOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewMakeLocklessCmd()) +} + +// NewMakeLocklessCmd constructs the "component make-lockless" command. +func NewMakeLocklessCmd() *cobra.Command { + filter := &components.ComponentFilter{} + cmd := &cobra.Command{ + Use: "make-lockless", + Short: "Select the lockless workflow and refresh upstream commits", + Long: `Add selected component names to lockless-components.txt beside the top-level +azldev.toml, then run the refresh-upstream-commit pipeline for those components. + +Existing entries, whitespace, and ordering are preserved. Missing names are +appended once, using exact component names rather than selection patterns. +Existing lock files are left untouched. Already-listed components are refreshed +again without duplicating their entries. + +Upstream components get generated commit TOML under base/upstream-commits/. +Include that directory in azldev.toml before component-specific configuration. +Non-upstream components are listed too, but do not contact upstream providers. +The refresh pipeline removes their generated commit TOML if one exists. + +If refreshing fails, the updated selection remains in place and successful +component pins are still saved. Fix the failure and rerun this command.`, + Example: ` azldev comp make-lockless -p curl + azldev comp make-lockless curl rust-asyncband + azldev comp make-lockless -g core + azldev comp make-lockless -a`, + RunE: azldev.RunFuncWithExtraArgs(func(env *azldev.Env, args []string) (interface{}, error) { + filter.ComponentNamePatterns = append(args, filter.ComponentNamePatterns...) + + return MakeComponentsLockless(env, filter) + }), + ValidArgsFunction: components.GenerateComponentNameCompletions, + Annotations: map[string]string{ + azldev.CommandAnnotationPermissiveConfig: "true", + }, + } + components.AddComponentFilterOptionsToCommand(cmd, filter) + + return cmd +} + +// MakeComponentsLockless selects the lockless workflow and refreshes selected component pins. +func MakeComponentsLockless( + env *azldev.Env, filter *components.ComponentFilter, +) ([]RefreshUpstreamCommitResult, error) { + if env.ProjectDir() == "" { + return nil, errors.New("no project directory configured; cannot select lockless components") + } + + filter.SkipLockValidation = true + resolver := components.NewResolver(env) + resolver.SuppressLockWarnings = true + + resolved, err := resolver.FindComponents(filter) + if err != nil { + return nil, fmt.Errorf("resolving components:\n%w", err) + } + + if resolved.Len() == 0 { + if filter.IncludeAllComponents { + return nil, nil + } + + return nil, errors.New("no components matched the filter") + } + + names := slices.Clone(resolved.Names()) + slices.Sort(names) + + if env.DryRun() { + slog.Info("Dry run; would select lockless components and refresh upstream commits", "components", names) + + return nil, nil + } + + if err := addLocklessComponents(env, names); err != nil { + return nil, err + } + + return RefreshUpstreamCommits(env, &RefreshUpstreamCommitOptions{ComponentFilter: *filter}) +} + +func addLocklessComponents(env *azldev.Env, names []string) error { + selected, err := projectconfig.ReadLocklessComponents(env.FS(), env.ProjectDir()) + if err != nil { + return fmt.Errorf("reading lockless component selection:\n%w", err) + } + + path := filepath.Join(env.ProjectDir(), "lockless-components.txt") + + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("reading %#q:\n%w", path, err) + } + + var additions strings.Builder + + for _, name := range names { + if !selected[name] { + additions.WriteString(name + "\n") + selected[name] = true + } + } + + if additions.Len() > 0 { + if len(content) > 0 && content[len(content)-1] != '\n' { + content = append(content, '\n') + } + + content = append(content, additions.String()...) + if err := fileutils.WriteFile(env.FS(), path, content, fileperms.PublicFile); err != nil { + return fmt.Errorf("writing %#q:\n%w", path, err) + } + } + + env.Config().LocklessComponents = selected + + return nil +} diff --git a/internal/app/azldev/cmds/component/makelockless_test.go b/internal/app/azldev/cmds/component/makelockless_test.go new file mode 100644 index 000000000..9d43cbeab --- /dev/null +++ b/internal/app/azldev/cmds/component/makelockless_test.go @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component_test + +import ( + "errors" + "os/exec" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + componentcmds "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/lockfile" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/upstreamcommit" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const makeLocklessTestPath = "/project/lockless-components.txt" + +func TestNewMakeLocklessCmd(t *testing.T) { + cmd := componentcmds.NewMakeLocklessCmd() + assert.Equal(t, "make-lockless", cmd.Use) + assert.NotNil(t, cmd.RunE) + assert.NotNil(t, cmd.Flags().Lookup("component")) + assert.NotNil(t, cmd.Flags().Lookup("all-components")) + assert.NotNil(t, cmd.Flags().Lookup("component-group")) + assert.Contains(t, cmd.Annotations, azldev.CommandAnnotationPermissiveConfig) +} + +func TestMakeLocklessCmd_NoComponents(t *testing.T) { + env := testutils.NewTestEnv(t) + cmd := componentcmds.NewMakeLocklessCmd() + cmd.SetArgs([]string{"nonexistent-component"}) + require.ErrorContains(t, cmd.ExecuteContext(env.Env), "component not found") + exists, err := fileutils.Exists(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.False(t, exists) +} + +func TestMakeComponentsLockless_SelectsAndRefreshes(t *testing.T) { + for _, testCase := range []struct { + name, existing, want string + }{ + {name: "missing file", want: "curl\nrust-asyncband\n"}, + {name: "missing newline", existing: "rust-asyncband", want: "rust-asyncband\ncurl\n"}, + {name: "existing whitespace", existing: "\t rust-asyncband \r\n\n", want: "\t rust-asyncband \r\n\ncurl\n"}, + {name: "already selected", existing: "curl\nrust-asyncband\n", want: "curl\nrust-asyncband\n"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + gitCalls := setupMockGitWithCounter(env, "new-upstream-commit") + for _, name := range []string{"curl", "rust-asyncband", "vim"} { + addRefreshUpstreamComponent(env, name) + } + + lock := lockfile.New() + lock.UpstreamCommit = "old-locked-commit" + env.WriteLock(t, "curl", lock) + previousLock, err := fileutils.ReadFile(env.TestFS, "/project/locks/curl.lock") + require.NoError(t, err) + + if testCase.existing != "" { + require.NoError(t, fileutils.WriteFile(env.TestFS, makeLocklessTestPath, + []byte(testCase.existing), fileperms.PublicFile)) + } + + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + require.NoError(t, store.Save("vim", "unselected-pin")) + + filter := &components.ComponentFilter{ComponentNamePatterns: []string{"rust-*", "curl"}} + _, err = componentcmds.MakeComponentsLockless(env.Env, filter) + require.NoError(t, err) + assert.Positive(t, gitCalls.Load()) + + for _, name := range []string{"curl", "rust-asyncband"} { + assert.True(t, env.Env.ComponentWithoutLockfile(name)) + pin, exists, err := store.Get(name) + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "new-upstream-commit", pin) + } + + pin, exists, err := store.Get("vim") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "unselected-pin", pin) + + content, err := fileutils.ReadFile(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.Equal(t, testCase.want, string(content)) + + currentLock, err := fileutils.ReadFile(env.TestFS, "/project/locks/curl.lock") + require.NoError(t, err) + assert.Equal(t, previousLock, currentLock) + + gitCalls.Store(0) + + _, err = componentcmds.MakeComponentsLockless(env.Env, filter) + require.NoError(t, err) + assert.Positive(t, gitCalls.Load(), "already-listed components must still refresh") + + content, err = fileutils.ReadFile(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.Equal(t, testCase.want, string(content)) + }) + } +} + +func TestMakeComponentsLockless_NonUpstream(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.Components["local-component"] = projectconfig.ComponentConfig{ + Name: "local-component", + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal, Path: "local-component.spec"}, + } + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + require.NoError(t, store.Save("local-component", "obsolete-pin")) + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected upstream command: %v", cmd.Args) + + return "", nil + } + cmd := componentcmds.NewMakeLocklessCmd() + cmd.SetArgs([]string{"-p", "local-component"}) + require.NoError(t, cmd.ExecuteContext(env.Env)) + content, err := fileutils.ReadFile(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.Equal(t, "local-component\n", string(content)) + + _, exists, err := store.Get("local-component") + require.NoError(t, err) + assert.False(t, exists) +} + +func TestMakeComponentsLockless_RefreshFailureKeepsSelection(t *testing.T) { + env := testutils.NewTestEnv(t) + addRefreshUpstreamComponent(env, "curl") + setupMockGit(env, "unused") + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return "", errors.New("upstream unavailable") + } + _, err := componentcmds.MakeComponentsLockless(env.Env, + &components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}) + require.Error(t, err) + content, err := fileutils.ReadFile(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.Equal(t, "curl\n", string(content)) + assert.True(t, env.Env.ComponentWithoutLockfile("curl")) +} + +func TestMakeComponentsLockless_InvalidSelectionFile(t *testing.T) { + env := testutils.NewTestEnv(t) + addRefreshUpstreamComponent(env, "curl") + require.NoError(t, fileutils.MkdirAll(env.TestFS, makeLocklessTestPath)) + _, err := componentcmds.MakeComponentsLockless(env.Env, + &components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}) + require.ErrorContains(t, err, "expected a file") + assert.False(t, env.Env.ComponentWithoutLockfile("curl")) +} + +func TestMakeComponentsLockless_DryRun(t *testing.T) { + env := testutils.NewTestEnv(t) + addRefreshUpstreamComponent(env, "curl") + dryEnv := azldev.NewEnv(t.Context(), azldev.EnvOptions{ + ProjectDir: "/project", Config: env.Config, Interfaces: env.TestInterfaces, + EventListener: env.EventListener, DryRunnable: azldev.NewAppDryRunnable(true), + }) + _, err := componentcmds.MakeComponentsLockless(dryEnv, + &components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}) + require.NoError(t, err) + exists, err := fileutils.Exists(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.False(t, exists) + assert.False(t, dryEnv.ComponentWithoutLockfile("curl")) +} From 421f75d9caf301c9565ad802026dd3687643c60e Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 03:28:46 +0000 Subject: [PATCH 28/70] fix(spec): preserve Release field alignment when updating values Retain the original Release tag prefix and its spaces or tabs in both spec editors rather than replacing the separator with a single space. Keep existing formatting behavior for other tags unchanged. This preserves lockless Release alignment through overlays, static and baserelease carryover, autorelease base edits, and final publication. Cover differently aligned conditional tags, tabs, zero spacing, autorelease edits, and the overlay-to-bump render pipeline. Update the lockless workflow guide and emitted agent skills. Verify the installed rpmdev-bumpspec retains alignment during its numeric bump. Validation: mage fix all, mage unit, mage check all, and mage docs passed. Unit tests also passed under mage all; Docker-dependent scenarios remain blocked. Emitted-skill install/show smoke checks passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 3 ++ .../app/azldev/agentskill/agentskill_test.go | 2 + .../azldev/agentskill/content/azldev.md.tmpl | 1 + .../content/refresh-upstream-commit.md.tmpl | 1 + .../content/withoutlockfile/azldev.md.tmpl | 1 + .../render_lockless_internal_test.go | 15 +++++--- .../render_release_carryover_internal_test.go | 30 ++++++++++++++- internal/rpm/spec/edit_test.go | 38 ++++++++++--------- internal/rpm/spec/editor.go | 14 +++++++ internal/rpm/spec/legacy_edit.go | 2 +- internal/rpm/spec/structural_edit.go | 2 +- 11 files changed, 83 insertions(+), 26 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 3c3113a1f..cba548dd4 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -253,6 +253,9 @@ Macro-file generation, existing overlays, and `sources` manifest updates run before release/changelog carryover and the final `rpmdev-bumpspec` invocation. Carryover and version-reset decisions therefore use the working spec after overlays; release/changelog edits made by overlays remain subject to those rules. +Release edits preserve the existing spaces and tabs between `Release:` and its +value, including during overlays and carryover, so aligned spec headers stay aligned +with either spec editor. Rendered output retains all prepared dist-git files except `.git`, including unreferenced sidecars, hidden files, and subdirectories. Lockless rendering does not run `spectool` or filter files; `render.skip-file-filter` diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 4fbf246ee..985c60ed0 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -190,6 +190,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "`update `") assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") assert.Contains(t, doc, "`azldev comp make-lockless`") + assert.Contains(t, doc, "spaces and tabs after `Release:`") } doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) @@ -224,6 +225,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "`update `") assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") assert.Contains(t, doc, "`azldev comp make-lockless`") + assert.Contains(t, doc, "spaces and tabs after `Release:`") assert.NotContains(t, doc, "git commit --amend") } diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 2ab67f373..41224bc0e 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -45,6 +45,7 @@ For listed upstream components, rendering writes macros, applies overlays, and updates `sources` before carrying release/changelog state from the committed output and running host `rpmdev-bumpspec`, without synthetic history or its legacy release adjustment. Lockless rendering +preserves the existing spaces and tabs after `Release:` when editing its value, and compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors without replacing existing output. The flag records host `date -Is` output in diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 877517446..c9c5af3fe 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -75,6 +75,7 @@ when writing the macros file; that step never repeats the provenance query. After reading render history and collecting upstream changes, rendering writes the macros file, applies overlays, and updates `sources` before carrying release/changelog state and running the final `rpmdev-bumpspec`. +Release edits preserve the existing spaces and tabs after `Release:`. Version detection and carryover operate on the working spec after overlays. Rendering compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 8f2d0853a..ebb4a4679 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -41,6 +41,7 @@ are still refreshed; a refresh failure leaves the list updated. For upstream components, rendering writes macros, applies overlays, and updates `sources` before carrying release/changelog state from the committed output and running host `rpmdev-bumpspec`. +Release edits preserve the existing spaces and tabs after `Release:`. Rendering compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors without replacing existing output. The flag records host `date -Is` output in diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 3d362311a..36da87414 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -1000,6 +1000,8 @@ func TestPrepareComponentSources_LocalAndLockedSkipNewLifecycle(t *testing.T) { } func TestPrepareComponentSources_OverlaysBeforeReleaseCarryover(t *testing.T) { + const releaseTag = "Release: " + for _, testCase := range []struct { name, comment, wantComment string explicit, noGitCommit bool @@ -1059,8 +1061,11 @@ upstream-commit = "aaaaaaa" manager := sourceproviders_test.NewMockSourceManager(ctrl) manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).DoAndReturn( func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + upstream := strings.Replace(renderTestSpec("2", "1", "* Upstream entry\n"), + "Release: ", releaseTag, 1) + return fileutils.WriteFile(env.TestFS, filepath.Join(dir, "curl.spec"), - []byte(renderTestSpec("2", "1", "* Upstream entry\n")), fileperms.PublicFile) + []byte(upstream), fileperms.PublicFile) }) originalPreparer := newSourcePreparer @@ -1096,7 +1101,7 @@ upstream-commit = "aaaaaaa" content, err := fileutils.ReadFile(env.TestFS, path) require.NoError(t, err) assert.Contains(t, string(content), "Summary: Overlaid summary\n") - assert.Contains(t, string(content), "Release: 9\n") + assert.Contains(t, string(content), releaseTag+"9\n") assert.Contains(t, string(content), "%changelog\n"+oldChangelog) assert.Contains(t, string(content), "%{load:%{_sourcedir}/curl.azl.macros}") assert.Contains(t, string(content), "Source9999") @@ -1110,7 +1115,7 @@ upstream-commit = "aaaaaaa" assert.Contains(t, string(manifest), "SHA512 (extra-source.tar.gz) = abc123") return fileutils.WriteFile(env.TestFS, path, - []byte(strings.Replace(string(content), "Release: 9\n", "Release: 10\n", 1)), fileperms.PublicFile) + []byte(strings.Replace(string(content), releaseTag+"9\n", releaseTag+"10\n", 1)), fileperms.PublicFile) } results := make([]*RenderResult, 1) prepared := parallelPrepare(env.Env, nil, []components.Component{comp}, "/staging", &RenderOptions{ @@ -1132,7 +1137,7 @@ upstream-commit = "aaaaaaa" content, err := fileutils.ReadFile(env.TestFS, "/project/SPECS/c/curl/curl.spec") require.NoError(t, err) - assert.Contains(t, string(content), "Release: 10\n") + assert.Contains(t, string(content), releaseTag+"10\n") var gitCommands []string @@ -1167,7 +1172,7 @@ upstream-commit = "aaaaaaa" content, err = fileutils.ReadFile(env.TestFS, "/project/SPECS/c/curl/curl.spec") require.NoError(t, err) - assert.Contains(t, string(content), "Release: 10\n") + assert.Contains(t, string(content), releaseTag+"10\n") }) } } diff --git a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go index 6becbbdc2..90fb9f34d 100644 --- a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go +++ b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go @@ -58,6 +58,7 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { path = "/working/curl.spec" oldChangelog = "* Old downstream entry\n" newRelease = "%{baserelease}.upstream%{?dist}" + releaseTag = "Release:\t " ) oldSpec := testCase.definition + "\n" + @@ -79,6 +80,7 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { // Upstream context and Release differ; only the macro definition is carried over. upstream := "# New upstream header\n%define baserelease 17\n" + renderTestSpec(testCase.version, newRelease, "* Upstream entry\n") + upstream = strings.Replace(upstream, "Release: ", releaseTag, 1) require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(upstream), fileperms.PublicFile)) env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { @@ -100,7 +102,7 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { } assert.Contains(t, string(content), definition+"\n") - assert.Contains(t, string(content), "Release: "+newRelease+"\n") + assert.Contains(t, string(content), releaseTag+newRelease+"\n") assert.Contains(t, string(content), "%changelog\n"+wantChangelog) return fileutils.WriteFile(env.TestFS, path, @@ -121,6 +123,7 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { content, err := fileutils.ReadFile(env.TestFS, path) require.NoError(t, err) assert.Contains(t, string(content), "baserelease "+testCase.wantBumped+"\n") + assert.Contains(t, string(content), releaseTag+newRelease+"\n") assert.Equal(t, oldSpec, history.spec) }) } @@ -128,6 +131,31 @@ func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { } } +func TestPrepareLocklessSpec_PreservesAutoreleaseSpacing(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + for _, spacing := range []string{" ", "\t \t", ""} { + t.Run(spacing, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + const path = "/working/curl.spec" + + upstream := strings.Replace(renderTestSpec("1", "%autorelease -b 2 -e test", "%autochangelog\n"), + "Release: ", "Release:"+spacing, 1) + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(upstream), fileperms.PublicFile)) + + err := prepareLocklessSpec(env.Env, &renderAutospecStub{base: "9"}, + "/working", "curl", editor, lastRender{}) + require.NoError(t, err) + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Contains(t, string(content), "Release:"+spacing+"%autorelease -b 9 -e test\n") + }) + } + }) + } +} + func TestCarryStaticRenderRelease(t *testing.T) { for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { t.Run(string(editor), func(t *testing.T) { diff --git a/internal/rpm/spec/edit_test.go b/internal/rpm/spec/edit_test.go index 5fff1d154..6dbca0a13 100644 --- a/internal/rpm/spec/edit_test.go +++ b/internal/rpm/spec/edit_test.go @@ -638,26 +638,28 @@ Name: value } func TestUpdateExistingTagUpdatesRepeatedConditionalTags(t *testing.T) { - specFile, err := spec.OpenSpec(strings.NewReader(`Name: example -%if 0 -Release: 1 -%else -Release: 2 -%endif -`), spec.WithEditor(spec.EditorStructural)) - require.NoError(t, err) + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + input := []string{ + "Name: example", + "%if 0", + "Release: 1", + "%else", + "Release:\t 2", + "%endif", + } + specFile, err := spec.OpenSpec(strings.NewReader(strings.Join(input, "\n")+"\n"), spec.WithEditor(editor)) + require.NoError(t, err) + require.NoError(t, specFile.UpdateExistingTag("", "Release", "3%{?dist}")) - require.NoError(t, specFile.UpdateExistingTag("", "Release", "3%{?dist}")) + var actual bytes.Buffer + require.NoError(t, specFile.Serialize(&actual)) - var actual bytes.Buffer - require.NoError(t, specFile.Serialize(&actual)) - assert.Equal(t, `Name: example -%if 0 -Release: 3%{?dist} -%else -Release: 3%{?dist} -%endif -`, actual.String()) + input[2] = "Release: 3%{?dist}" + input[4] = "Release:\t 3%{?dist}" + assert.Equal(t, strings.Join(input, "\n")+"\n", actual.String()) + }) + } } func TestRemoveTag(t *testing.T) { diff --git a/internal/rpm/spec/editor.go b/internal/rpm/spec/editor.go index 78967de13..b2041bf3b 100644 --- a/internal/rpm/spec/editor.go +++ b/internal/rpm/spec/editor.go @@ -6,6 +6,7 @@ package spec import ( "fmt" "io" + "strings" "time" ) @@ -72,6 +73,19 @@ type Spec struct { editor specEditor } +func updatedTagLine(original, tag, value string) string { + if strings.EqualFold(tag, "Release") { + prefix, rest, found := strings.Cut(original, ":") + if found { + spacing := rest[:len(rest)-len(strings.TrimLeft(rest, " \t"))] + + return prefix + ":" + spacing + value + } + } + + return fmt.Sprintf("%s: %s", tag, value) +} + func (s *Spec) activeEditor() specEditor { if s.editor == nil { s.editor = &legacySpec{} diff --git a/internal/rpm/spec/legacy_edit.go b/internal/rpm/spec/legacy_edit.go index 3f530beb6..c5c6544e7 100644 --- a/internal/rpm/spec/legacy_edit.go +++ b/internal/rpm/spec/legacy_edit.go @@ -60,7 +60,7 @@ func (s *legacySpec) UpdateExistingTag(packageName string, tag string, value str return nil } - ctx.ReplaceLine(fmt.Sprintf("%s: %s", tag, value)) + ctx.ReplaceLine(updatedTagLine(ctx.Target.Line.Text, tag, value)) updated = true diff --git a/internal/rpm/spec/structural_edit.go b/internal/rpm/spec/structural_edit.go index 2b8dc2805..3552b06e1 100644 --- a/internal/rpm/spec/structural_edit.go +++ b/internal/rpm/spec/structural_edit.go @@ -52,7 +52,7 @@ func (s *structuralSpec) UpdateExistingTag(packageName string, tag string, value return nil } - line.Replace(fmt.Sprintf("%s: %s", tag, value)) + line.Replace(updatedTagLine(line.Text, tag, value)) updated = true From 2c91341aa987a3929cfd061ae8b59aa9c143c512 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 05:23:40 +0000 Subject: [PATCH 29/70] fix(component): omit redundant upstream pin TOML tables Emit only the component spec table in generated upstream pin TOML files. Preserve TOML key quoting and compatibility with existing parent tables. Cover compact output, quoted component names, and legacy-file loading. Document the generated format in the lockless workflow guide. Validation: mage fix all, mage unit, and mage check all passed. mage all built successfully, but scenarios remain blocked by SELinux and Docker. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 12 ++++++-- internal/upstreamcommit/store.go | 5 +++ internal/upstreamcommit/store_test.go | 32 ++++++++++++++++++-- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index cba548dd4..17ced5b16 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -88,8 +88,16 @@ includes = [ ``` Generated files hold only `spec.upstream-commit`; the component's own TOML -supplies the source type and everything else. Because a single file may hold a -partial component definition for a lockless component, its validation runs after +supplies the source type and everything else. Newly written files use one table, +`[components..spec]`, without separate parent-table headers: + +```toml +[components.curl.spec] +upstream-commit = 'abcdef1234567' +``` + +Existing files with explicit parent tables remain readable. Because a single file +may hold a partial component definition for a lockless component, its validation runs after all config files have been merged. Unlisted components retain normal additive merging and per-file validation. diff --git a/internal/upstreamcommit/store.go b/internal/upstreamcommit/store.go index 434900290..f0abbb541 100644 --- a/internal/upstreamcommit/store.go +++ b/internal/upstreamcommit/store.go @@ -6,6 +6,7 @@ package upstreamcommit import ( + "bytes" "errors" "fmt" "log/slog" @@ -162,6 +163,10 @@ func (s *Store) Save(componentName, upstreamCommit string) error { return fmt.Errorf("serializing upstream commit TOML %#q:\n%w", path, err) } + // Only the spec table has values; omit the encoder's two empty parent headers. + _, data, _ = bytes.Cut(data, []byte("\n")) + _, data, _ = bytes.Cut(data, []byte("\n")) + data = append([]byte(generatedFileHeader), data...) if err := fileutils.MkdirAll(s.fs, s.dir); err != nil { diff --git a/internal/upstreamcommit/store_test.go b/internal/upstreamcommit/store_test.go index 923d56bd1..7bebc5735 100644 --- a/internal/upstreamcommit/store_test.go +++ b/internal/upstreamcommit/store_test.go @@ -5,6 +5,7 @@ package upstreamcommit_test import ( "path/filepath" + "strings" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" @@ -29,8 +30,6 @@ func TestStoreSaveAndGet(t *testing.T) { assert.Equal(t, `# This file was generated by 'azldev component refresh-upstream-commit' # Do not edit this file, changes will be lost # For more details see 'azldev component refresh-upstream-commit --help' -[components] -[components.bash] [components.bash.spec] upstream-commit = 'abcdef1234567' `, string(data)) @@ -47,6 +46,27 @@ upstream-commit = 'abcdef1234567' assert.Equal(t, "1234567abcdef", commit) } +func TestStoreSaveSingleTableWithQuotedComponentNames(t *testing.T) { + for _, name := range []string{"rust-asyncband", "libstdc++", "python3.12", `component"quote`} { + t.Run(name, func(t *testing.T) { + ctx := testctx.NewCtx() + store := upstreamcommit.NewStore(ctx.FS(), testStoreDir) + require.NoError(t, store.Save(name, "abcdef1234567")) + path, err := store.Path(name) + require.NoError(t, err) + data, err := fileutils.ReadFile(ctx.FS(), path) + require.NoError(t, err) + assert.Equal(t, 1, strings.Count(string(data), "\n[")) + assert.Contains(t, string(data), ".spec]\n") + + commit, exists, err := store.Get(name) + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "abcdef1234567", commit) + }) + } +} + func TestStoreSaveReplacesExistingGeneratedContent(t *testing.T) { ctx := testctx.NewCtx() store := upstreamcommit.NewStore(ctx.FS(), testStoreDir) @@ -54,6 +74,8 @@ func TestStoreSaveReplacesExistingGeneratedContent(t *testing.T) { require.NoError(t, fileutils.MkdirAll(ctx.FS(), testStoreDir)) require.NoError(t, fileutils.WriteFile(ctx.FS(), path, []byte(` +[components] +[components.bash] [components.bash.spec] type = "local" path = "bash.spec" @@ -63,6 +85,11 @@ upstream-commit = "oldcommit" defines = { stale = "value" } `), fileperms.PublicFile)) + commit, exists, err := store.Get("bash") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "oldcommit", commit) + require.NoError(t, store.Save("bash", "abcdef1234567")) data, err := fileutils.ReadFile(ctx.FS(), path) @@ -71,6 +98,7 @@ defines = { stale = "value" } assert.NotContains(t, string(data), "path") assert.NotContains(t, string(data), "defines") assert.Contains(t, string(data), "upstream-commit = 'abcdef1234567'") + assert.Equal(t, 1, strings.Count(string(data), "\n[")) } func TestStoreGetMissing(t *testing.T) { From cb845a56f722f6700f3fbc1e5c324845a479c775 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 20:04:24 +0000 Subject: [PATCH 30/70] fix(component): prepare changelog and release before overlays Prepare changelog state before release carryover in the fetched checkout. Run both before writing macros, applying overlays, and updating sources. Base version-dependent carryover on the upstream spec before overlays. Allow overlays to override the prepared release and changelog without reapplying carryover afterwards. Update regression tests, CLI help, user documentation, and emitted skills. Validated with mage unit, mage check all, mage docs, and agent smoke tests. A current-code render of rust-git2_0.19 also succeeded. Full scenarios and snapshot updates remain blocked by the host Docker/SELinux setup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 37 ++++----- .../reference/cli/azldev_component_render.md | 5 +- .../app/azldev/agentskill/agentskill_test.go | 10 ++- .../azldev/agentskill/content/azldev.md.tmpl | 8 +- .../content/refresh-upstream-commit.md.tmpl | 30 ++++---- .../content/withoutlockfile/azldev.md.tmpl | 8 +- internal/app/azldev/cmds/component/render.go | 27 ++----- .../azldev/cmds/component/render_lockless.go | 65 ++++++++++------ .../render_lockless_internal_test.go | 75 +++++++++++++++---- 9 files changed, 164 insertions(+), 101 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 17ced5b16..a98b012bc 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -155,8 +155,22 @@ provenance is disabled or the component is not sourced from Fedora upstream. The renderer then finds the existing committed output under `//`, ignoring -uncommitted spec changes. After reading the working spec's post-overlay -`Release:` and `Version:`, it compares that version with the committed spec's +uncommitted spec changes. It prepares the changelog first, then Release, before +writing the macros file, applying overlays, or updating `sources`. + +Changelog handling is independent of version changes. For non-`%autorelease` +specs, the committed inline `%changelog` section is preserved. For an initial +`%autorelease` render, host `rpmautospec generate-changelog` creates the complete +upstream `changelog`, replacing any existing file. For legacy-marker output, the +working `changelog` is replaced with the committed inline `%changelog` body, +excluding the section header and leading or trailing comment lines starting with +`##`; interior comments and whitespace are preserved. For modern `%autorelease` +output, the committed `changelog` file replaces the working file byte-for-byte, +or is created if absent from the working checkout. A missing committed +`changelog` fails rendering. + +Release preparation reads the upstream spec's `Release:` and `Version:` before +overlays, and compares that version with the committed spec's `Version:`, independently of upstream pin changes. Static release carryover runs **only when committed output exists and the version is unchanged**; otherwise the working static release and `baserelease` definition are left as supplied by @@ -198,18 +212,6 @@ and removes any existing working base option, using the `%autorelease` default. Other macro options are preserved. Unchanged-version modern output still carries an explicitly committed `-b 1` as-is. -Changelog handling follows release preparation and runs even when the version -is unchanged. For non-`%autorelease` specs, the committed inline `%changelog` -section is preserved. For an initial `%autorelease` render, host -`rpmautospec generate-changelog` creates the complete upstream `changelog`, -replacing any existing file. For legacy-marker output, the working `changelog` -is replaced with the committed inline `%changelog` body, excluding the section -header and leading or trailing comment lines starting with `##`; interior -comments and whitespace are preserved. For modern `%autorelease` output, the -committed `changelog` file replaces the working file byte-for-byte, or is created -if absent from the working checkout. A missing committed `changelog` fails -rendering. - The last commit touching the component output is the **last render commit**. The renderer fully loads TOML, including includes and component defaults, at that commit and at the current project commit to compare `spec.upstream-commit`. @@ -258,9 +260,10 @@ azldev comp render -p curl --changelog-comment "Fix curl patches" ``` Macro-file generation, existing overlays, and `sources` manifest updates run -before release/changelog carryover and the final `rpmdev-bumpspec` invocation. -Carryover and version-reset decisions therefore use the working spec after -overlays; release/changelog edits made by overlays remain subject to those rules. +after changelog and Release preparation, but before comparison and the final +`rpmdev-bumpspec` invocation. Overlays can override the prepared Release and +changelog; carryover is not repeated afterwards. Version changes made by overlays +do not alter the earlier carryover decision. Release edits preserve the existing spaces and tabs between `Release:` and its value, including during overlays and carryover, so aligned spec headers stay aligned with either spec editor. diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 5b677b375..a59aa469f 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -26,8 +26,9 @@ its lock file, using configured upstream commits and their TOML history instead. All other components use lock files. A missing or empty file means all components use lock files. -For lockless upstream components, write macros, apply overlays, and update sources -before carrying release and changelog state from the last committed render. +For lockless upstream components, prepare the changelog, then Release, using the +upstream spec and last committed render before writing macros, applying overlays, +and updating sources. Overlays can override the prepared release and changelog. Compare output; unchanged renders require --no-change-rebuild to record a timestamp. Then run rpmdev-bumpspec -c with --changelog-comment (default: "update "). Commit only that directory with the same subject and upstream changes as the body. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 985c60ed0..873c350a4 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -179,7 +179,9 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "never uses mock or `rpmautospec process-distgit`") assert.Contains(t, doc, "all prepared dist-git files except `.git`") assert.NotContains(t, doc, "host `spectool` filtering") - assert.Contains(t, doc, "updates `sources` before carrying release/changelog state") + assert.Contains(t, doc, "prepares the changelog before Release") + assert.Contains(t, doc, "Version detection and carryover use the upstream spec before overlays") + assert.Contains(t, doc, "Overlays can override the prepared release and changelog") assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") assert.Contains(t, doc, "Unchanged lockless upstream renders require `--no-change-rebuild`") assert.Contains(t, doc, "host `date -Is` output") @@ -207,14 +209,16 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "When the version changes, rendering calculates the upstream numeric base and sets `-b`") assert.Contains(t, doc, "The committed changelog is still carried over, not regenerated") assert.Contains(t, doc, "A calculated base of `1` omits `-b`") - assert.Contains(t, doc, "Changelog handling follows release preparation and runs even when the version") + assert.Contains(t, doc, "Changelog handling precedes release preparation and is independent of version") assert.Contains(t, doc, "retains `%autorelease`, `%autochangelog`") assert.Contains(t, doc, "effective upstream distro's `dist-git-branch`") assert.Contains(t, doc, "fetches that historical branch before comparing commits") assert.Contains(t, doc, "All prepared dist-git files except `.git` are retained") assert.Contains(t, doc, "immediately after checking out the pinned upstream commit") assert.Contains(t, doc, "that step never repeats the provenance query") - assert.Contains(t, doc, "the macros file, applies overlays, and updates `sources` before carrying") + assert.Contains(t, doc, "the changelog before Release, then writes the macros file") + assert.Contains(t, doc, "Version detection and carryover use the upstream spec before overlays") + assert.Contains(t, doc, "Overlays can override the prepared release and changelog") assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") assert.Contains(t, doc, "Unchanged lockless upstream renders require `--no-change-rebuild`") assert.Contains(t, doc, "host `date -Is` output") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 41224bc0e..36abaa4d3 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -41,9 +41,11 @@ generated upstream-commit TOML instead. All others use locks. then runs the refresh pipeline. Existing list contents and lock files are retained. Already-listed components are still refreshed; a refresh failure leaves the list updated. -For listed upstream components, rendering writes macros, applies overlays, and -updates `sources` before carrying release/changelog state from the committed -output and running host `rpmdev-bumpspec`, without synthetic history or its legacy +For listed upstream components, rendering prepares the changelog before Release, +then writes macros, applies overlays, and updates `sources`. +Version detection and carryover use the upstream spec before overlays. +Overlays can override the prepared release and changelog. +Rendering then runs host `rpmdev-bumpspec`, without synthetic history or its legacy release adjustment. Lockless rendering preserves the existing spaces and tabs after `Release:` when editing its value, and compares output before the final release bump; `--check-only` stops before that bump. diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index c9c5af3fe..accac4347 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -72,11 +72,13 @@ runs here only for an upstream `%autorelease` whose `fedora_upstream_release` is not already supplied by `build.defines`. The captured values are reused when writing the macros file; that step never repeats the provenance query. -After reading render history and collecting upstream changes, rendering writes -the macros file, applies overlays, and updates `sources` before carrying -release/changelog state and running the final `rpmdev-bumpspec`. +After reading render history and collecting upstream changes, rendering prepares +the changelog before Release, then writes the macros file, applies overlays, and +updates `sources` before comparison and the final `rpmdev-bumpspec`. Release edits preserve the existing spaces and tabs after `Release:`. -Version detection and carryover operate on the working spec after overlays. +Version detection and carryover use the upstream spec before overlays. +Overlays can override the prepared release and changelog. +Carryover is not repeated after overlays, even if they change the version. Rendering compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors without replacing existing output. The flag records host `date -Is` output in @@ -88,7 +90,16 @@ use Git to restore committed output, or leave no directory if none existed. Recovery discards uncommitted changes in that directory and keeps no backups. The comparison ignores the root `.git` without traversing or removing it. -After reading the working spec, rendering compares its `Version:` with the +Changelog handling precedes release preparation and is independent of version +changes. Static specs retain the committed inline `%changelog` section. +An initial `%autorelease` render generates the complete upstream changelog on +the host. Legacy-marker output supplies its committed inline `%changelog` body +as the working `changelog`, removing leading and trailing `##` comment lines +while preserving interior comments and whitespace. Modern autorelease output +replaces the working `changelog` with the committed file byte-for-byte. +A missing committed `changelog` fails rendering. + +Before overlays, rendering compares the upstream spec's `Version:` with the committed spec, independently of upstream pin changes. Static release carryover runs only when committed output exists and the version is unchanged. Otherwise the working static release @@ -118,14 +129,7 @@ A calculated base of `1` omits `-b`, removing any working base option while pres other macro options. This includes legacy expansion. Unchanged-version modern output still carries an explicitly committed `-b 1` as-is. -Changelog handling follows release preparation and runs even when the version -is unchanged. Static specs retain the committed inline `%changelog` section. -An initial `%autorelease` render generates the complete upstream changelog on -the host. Legacy-marker output supplies its committed inline `%changelog` body -as the working `changelog`, removing leading and trailing `##` comment lines -while preserving interior comments and whitespace. Modern autorelease output -replaces the working `changelog` with the committed file byte-for-byte. -A missing committed `changelog` fails rendering. The final host +The final host `rpmdev-bumpspec -c` uses the exact `--changelog-comment` value, defaulting to `update `. Upstream changes are not appended to this comment. All prepared dist-git files except `.git` are retained, diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index ebb4a4679..b671d4506 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -38,9 +38,11 @@ images assemble RPMs into bootable artifacts. Existing list contents and lock files are retained. Already-listed components are still refreshed; a refresh failure leaves the list updated. -For upstream components, rendering writes macros, applies overlays, and -updates `sources` before carrying release/changelog state from the committed -output and running host `rpmdev-bumpspec`. +For upstream components, rendering prepares the changelog before Release, +then writes macros, applies overlays, and updates `sources`. +Version detection and carryover use the upstream spec before overlays. +Overlays can override the prepared release and changelog. +Rendering then runs host `rpmdev-bumpspec`. Release edits preserve the existing spaces and tabs after `Release:`. Rendering compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 4a6f053f2..678dcf65e 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -78,8 +78,9 @@ its lock file, using configured upstream commits and their TOML history instead. All other components use lock files. A missing or empty file means all components use lock files. -For lockless upstream components, write macros, apply overlays, and update sources -before carrying release and changelog state from the last committed render. +For lockless upstream components, prepare the changelog, then Release, using the +upstream spec and last committed render before writing macros, applying overlays, +and updating sources. Overlays can override the prepared release and changelog. Compare output; unchanged renders require --no-change-rebuild to record a timestamp. Then run rpmdev-bumpspec -c with --changelog-comment (default: "update "). Commit only that directory with the same subject and upstream changes as the body. @@ -610,17 +611,14 @@ func prepareComponentSources( preparerOpts = append(preparerOpts, sources.WithMockProcessor(mockProcessor)) } - var ( - update *locklessRenderUpdate - history lastRender - ) + var update *locklessRenderUpdate if lockless && comp.GetConfig().Spec.SourceType == projectconfig.SpecSourceTypeUpstream { preparerOpts = append(preparerOpts, sources.WithFetchedSources( func(_ context.Context, fetched components.Component, dir string) error { var prepErr error //nolint:contextcheck // The callback context is the captured preparation env. - history, update, prepErr = prepareLocklessRender(env, fetched, dir, compOutputDir, specEditor) + update, prepErr = prepareLocklessRender(env, fetched, dir, compOutputDir, specEditor) return prepErr })) @@ -642,21 +640,6 @@ func prepareComponentSources( return nil, fmt.Errorf("preparing sources for %#q:\n%w", componentName, prepErr) } - return finishSourcePreparation(env, comp, componentDir, specEditor, history, update, lockless) -} - -func finishSourcePreparation( - env *azldev.Env, comp components.Component, componentDir string, specEditor spec.EditorMode, - history lastRender, update *locklessRenderUpdate, lockless bool, -) (*preparedComponent, error) { - componentName := comp.GetName() - if update != nil { - if err := prepareLocklessSpec(env, hostRenderProcessor{env: env}, componentDir, componentName, - specEditor, history); err != nil { - return nil, fmt.Errorf("preparing release and changelog for %#q:\n%w", componentName, err) - } - } - // Find the spec file so we can pass the filename to mock. specPath, specErr := findSpecFile(env.FS(), componentDir, componentName) if specErr != nil { diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 1307e9c60..20d4c1179 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -64,23 +64,23 @@ var openRenderRepo = openChangedRepo func prepareLocklessRender( env *azldev.Env, comp components.Component, dir, outputDir string, editor spec.EditorMode, -) (history lastRender, update *locklessRenderUpdate, err error) { +) (*locklessRenderUpdate, error) { repo, root, err := openRenderRepo(env) if err != nil { - return history, nil, err + return nil, err } path, err := repoRelPath(root, outputDir) if err != nil { - return history, nil, fmt.Errorf("lockless render output must be inside the project repository:\n%w", err) + return nil, fmt.Errorf("lockless render output must be inside the project repository:\n%w", err) } - history, err = readLastRender(env, repo, root, path, comp.GetName()) + history, err := readLastRender(env, repo, root, path, comp.GetName()) if err != nil { - return history, nil, fmt.Errorf("reading committed render state:\n%w", err) + return nil, fmt.Errorf("reading committed render state:\n%w", err) } - update = &locklessRenderUpdate{ + update := &locklessRenderUpdate{ repoRoot: root, path: path, subject: "update " + comp.GetName(), @@ -88,19 +88,23 @@ func prepareLocklessRender( _, opened, err := openLocklessSpec(env, dir, comp.GetName(), editor) if err != nil { - return history, nil, err + return nil, err } version, err := opened.GetLastTag("", "Version") if err != nil { - return history, nil, fmt.Errorf("reading upstream Version:\n%w", err) + return nil, fmt.Errorf("reading upstream Version:\n%w", err) } if err := collectRenderUpstreamChanges(env, dir, comp.GetName(), version, history, update); err != nil { - return history, nil, err + return nil, err } - return history, update, nil + if err := prepareLocklessSpec(env, hostRenderProcessor{env: env}, dir, comp.GetName(), editor, history); err != nil { + return nil, fmt.Errorf("preparing changelog and release for %#q:\n%w", comp.GetName(), err) + } + + return update, nil } func readLastRender( @@ -295,6 +299,15 @@ func prepareLocklessSpec( return err } + if err := prepareLocklessChangelog(env, autospec, dir, filepath.Base(path), original, history); err != nil { + return err + } + + opened, err = carryLocklessInlineChangelog(opened, history.spec, original, editor) + if err != nil { + return err + } + if !versionChanged || sources.ReleaseUsesAutorelease(original) { release, err := locklessRenderRelease(env, autospec, dir, filepath.Base(path), original, history, versionChanged) if err != nil { @@ -310,33 +323,41 @@ func prepareLocklessSpec( } } - if err := prepareLocklessChangelog(env, autospec, dir, filepath.Base(path), original, history); err != nil { - return err - } - - resultBytes, err := locklessSpecBytes(opened, history.spec, original, editor) - if err != nil { - return err + var result bytes.Buffer + if err := opened.Serialize(&result); err != nil { + return fmt.Errorf("serializing prepared upstream spec:\n%w", err) } - if err := fileutils.WriteFile(env.FS(), path, resultBytes, fileperms.PublicFile); err != nil { + if err := fileutils.WriteFile(env.FS(), path, result.Bytes(), fileperms.PublicFile); err != nil { return fmt.Errorf("writing prepared upstream spec:\n%w", err) } return nil } -func locklessSpecBytes(opened *spec.Spec, previous, release string, editor spec.EditorMode) ([]byte, error) { +func carryLocklessInlineChangelog( + opened *spec.Spec, previous, release string, editor spec.EditorMode, +) (*spec.Spec, error) { + if previous == "" || sources.ReleaseUsesAutorelease(release) { + return opened, nil + } + var result bytes.Buffer if err := opened.Serialize(&result); err != nil { return nil, fmt.Errorf("serializing upstream spec:\n%w", err) } - if previous != "" && !sources.ReleaseUsesAutorelease(release) { - return restoreRenderChangelog(result.Bytes(), []byte(previous), editor) + content, err := restoreRenderChangelog(result.Bytes(), []byte(previous), editor) + if err != nil { + return nil, err + } + + carried, err := spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor)) + if err != nil { + return nil, fmt.Errorf("parsing spec with committed changelog:\n%w", err) } - return result.Bytes(), nil + return carried, nil } func collectRenderUpstreamChanges( diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 36da87414..4d5f482f9 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -534,7 +534,7 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { assert.Equal(t, testCase.wantChanges, update.changes) if testCase.bootstrap { - assert.Equal(t, []string{"release", "changelog"}, stub.calls) + assert.Equal(t, []string{"changelog", "release"}, stub.calls) changelog, err := fileutils.ReadFile(env.TestFS, "/working/changelog") require.NoError(t, err) @@ -636,6 +636,11 @@ func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { stub := &renderAutospecStub{expand: func(dir, filename string) (string, error) { snapshot = dir + changelog, err := fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, carriedAutoreleaseChangelog, string(changelog), + "carry the changelog before expanding the committed release") + content, err := fileutils.ReadFile(env.TestFS, filepath.Join(dir, filename)) require.NoError(t, err) assert.Equal(t, history.spec, string(content)) @@ -865,7 +870,7 @@ func TestPrepareLocklessSpec_UnprocessedAutoreleaseFailure(t *testing.T) { lastRender{spec: committed, tree: tree}) require.ErrorContains(t, err, testCase.want) - if testCase.newVersion { + if testCase.newVersion && !testCase.missingFile && !testCase.missingTree { assert.Equal(t, []string{"release"}, stub.calls) } else { assert.Empty(t, stub.calls) @@ -876,7 +881,13 @@ func TestPrepareLocklessSpec_UnprocessedAutoreleaseFailure(t *testing.T) { assert.Equal(t, original, string(content)) content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") require.NoError(t, err) - assert.Equal(t, "upstream changelog", string(content)) + + wantChangelog := "committed changelog" + if testCase.missingFile || testCase.missingTree { + wantChangelog = "upstream changelog" + } + + assert.Equal(t, wantChangelog, string(content)) }) } } @@ -926,7 +937,7 @@ func TestPrepareLocklessSpec_CommittedAutoreleaseFailure(t *testing.T) { assert.Equal(t, original, string(content)) content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") require.NoError(t, err) - assert.Equal(t, "original changelog", string(content)) + assert.Equal(t, carriedAutoreleaseChangelog, string(content)) }) } } @@ -999,12 +1010,13 @@ func TestPrepareComponentSources_LocalAndLockedSkipNewLifecycle(t *testing.T) { } } -func TestPrepareComponentSources_OverlaysBeforeReleaseCarryover(t *testing.T) { +func TestPrepareComponentSources_ChangelogAndReleaseBeforeOverlays(t *testing.T) { const releaseTag = "Release: " for _, testCase := range []struct { - name, comment, wantComment string - explicit, noGitCommit bool + name, comment, wantComment string + upstreamVersion, overlayVersion, wantRelease string + explicit, noGitCommit, skipReleaseOverlay bool }{ {name: "default comment", wantComment: "update curl"}, { @@ -1016,23 +1028,52 @@ func TestPrepareComponentSources_OverlaysBeforeReleaseCarryover(t *testing.T) { name: "no Git commit", comment: "Prepare curl for review", wantComment: "Prepare curl for review", noGitCommit: true, }, + { + name: "version overlay cannot enable carryover", wantComment: "update curl", + upstreamVersion: "2", overlayVersion: "1", wantRelease: "1", skipReleaseOverlay: true, + }, + { + name: "version overlay cannot disable carryover", wantComment: "update curl", + upstreamVersion: "1", overlayVersion: "2", wantRelease: "9", skipReleaseOverlay: true, + }, } { t.Run(testCase.name, func(t *testing.T) { env := testutils.NewTestEnvWithoutLockfile(t) ctrl := gomock.NewController(t) + + upstreamVersion, overlayVersion, wantRelease := "2", "1", "2" + if testCase.skipReleaseOverlay { + upstreamVersion = testCase.upstreamVersion + overlayVersion = testCase.overlayVersion + wantRelease = testCase.wantRelease + } + + releaseNumber, err := strconv.Atoi(wantRelease) + require.NoError(t, err) + + wantBumped := strconv.Itoa(releaseNumber + 1) + config := &projectconfig.ComponentConfig{ Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, Build: projectconfig.ComponentBuildConfig{With: []string{"feature"}}, Overlays: []projectconfig.ComponentOverlay{ {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Release", Value: "2"}, - {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: "1"}, + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: overlayVersion}, {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Overlaid summary"}, + { + Type: projectconfig.ComponentOverlaySearchAndReplaceInSpec, SectionName: "%changelog", + Regex: "Downstream entry", Replacement: "Overlaid entry", + }, }, SourceFiles: []projectconfig.SourceFileReference{{ Filename: "extra-source.tar.gz", Hash: "abc123", HashType: fileutils.HashTypeSHA512, Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://example.com/extra-source.tar.gz"}, }}, } + if testCase.skipReleaseOverlay { + config.Overlays = config.Overlays[1:] + } + comp := components_testutils.NewMockComponent(ctrl) comp.EXPECT().GetName().AnyTimes().Return("curl") comp.EXPECT().GetConfig().AnyTimes().Return(config) @@ -1061,7 +1102,7 @@ upstream-commit = "aaaaaaa" manager := sourceproviders_test.NewMockSourceManager(ctrl) manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).DoAndReturn( func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { - upstream := strings.Replace(renderTestSpec("2", "1", "* Upstream entry\n"), + upstream := strings.Replace(renderTestSpec(upstreamVersion, "1", "* Upstream entry\n"), "Release: ", releaseTag, 1) return fileutils.WriteFile(env.TestFS, filepath.Join(dir, "curl.spec"), @@ -1101,8 +1142,9 @@ upstream-commit = "aaaaaaa" content, err := fileutils.ReadFile(env.TestFS, path) require.NoError(t, err) assert.Contains(t, string(content), "Summary: Overlaid summary\n") - assert.Contains(t, string(content), releaseTag+"9\n") - assert.Contains(t, string(content), "%changelog\n"+oldChangelog) + assert.Contains(t, string(content), releaseTag+wantRelease+"\n") + assert.Contains(t, string(content), "Version: "+overlayVersion+"\n") + assert.Contains(t, string(content), "%changelog\n* Overlaid entry\n") assert.Contains(t, string(content), "%{load:%{_sourcedir}/curl.azl.macros}") assert.Contains(t, string(content), "Source9999") @@ -1115,14 +1157,15 @@ upstream-commit = "aaaaaaa" assert.Contains(t, string(manifest), "SHA512 (extra-source.tar.gz) = abc123") return fileutils.WriteFile(env.TestFS, path, - []byte(strings.Replace(string(content), releaseTag+"9\n", releaseTag+"10\n", 1)), fileperms.PublicFile) + []byte(strings.Replace(string(content), releaseTag+wantRelease+"\n", + releaseTag+wantBumped+"\n", 1)), fileperms.PublicFile) } results := make([]*RenderResult, 1) prepared := parallelPrepare(env.Env, nil, []components.Component{comp}, "/staging", &RenderOptions{ OutputDir: "/project/SPECS", ChangelogComment: testCase.comment, changelogCommentExplicit: testCase.explicit, NoGitCommit: testCase.noGitCommit, }, results) - require.Len(t, prepared, 1) + require.Len(t, prepared, 1, "preparation results: %+v", results[0]) prep := prepared[0] prep.update.changes = "abc123 Fix upstream" @@ -1137,7 +1180,7 @@ upstream-commit = "aaaaaaa" content, err := fileutils.ReadFile(env.TestFS, "/project/SPECS/c/curl/curl.spec") require.NoError(t, err) - assert.Contains(t, string(content), releaseTag+"10\n") + assert.Contains(t, string(content), releaseTag+wantBumped+"\n") var gitCommands []string @@ -1172,7 +1215,7 @@ upstream-commit = "aaaaaaa" content, err = fileutils.ReadFile(env.TestFS, "/project/SPECS/c/curl/curl.spec") require.NoError(t, err) - assert.Contains(t, string(content), releaseTag+"10\n") + assert.Contains(t, string(content), releaseTag+wantBumped+"\n") }) } } @@ -1276,7 +1319,7 @@ func TestPrepareLocklessSpec_StopsOnAutospecFailure(t *testing.T) { stub := &renderAutospecStub{err: errors.New("autospec failed")} err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, lastRender{}) require.ErrorContains(t, err, "autospec failed") - assert.Equal(t, []string{"release"}, stub.calls) + assert.Equal(t, []string{"changelog"}, stub.calls) } func TestFinishLocklessSpec_TimestampFailures(t *testing.T) { From 8b33f04e7c47b9e4e284fefaf90649932146830c Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 20:33:31 +0000 Subject: [PATCH 31/70] feat(component): guard dirty lockless upstream renders Require a clean project Git repository before preparing any selected lockless upstream component. Reject staged changes, unstaged tracked changes, and untracked files across the whole repository, excluding Git-ignored files. Fail closed when querying Git status fails. Add --allow-uncommitted to explicitly bypass this preflight. Check-only and no-git-commit do not bypass it; locked and local components retain their existing behavior. Cover refusal before preparation, mixed selections, override wiring, query failures, and scope. Update the user guide, generated CLI reference, and emitted skills, including dirty-input iteration guidance. Validation: mage fix all, mage unit, mage check all, and mage docs passed. Real CLI smoke checks passed for ignored files, outside-project tracked and untracked changes, staged changes, and the explicit override. CLI help and full agent installation checks passed. Peak memory was 2.1 GiB. mage all and mage scenarioUpdate remain blocked by the host's Docker provider initialization failure under enforcing SELinux. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 16 ++ .../reference/cli/azldev_component_render.md | 7 + .../app/azldev/agentskill/agentskill_test.go | 5 + .../azldev/agentskill/content/azldev.md.tmpl | 8 + .../content/build-component.md.tmpl | 8 +- .../content/refresh-upstream-commit.md.tmpl | 11 +- .../content/withoutlockfile/azldev.md.tmpl | 8 + .../withoutlockfile/build-component.md.tmpl | 5 + internal/app/azldev/cmds/component/render.go | 21 ++- .../azldev/cmds/component/render_lockless.go | 35 +++++ .../component/render_modes_internal_test.go | 140 ++++++++++++++++++ .../app/azldev/cmds/component/render_test.go | 56 +++++++ 12 files changed, 311 insertions(+), 9 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index a98b012bc..18b538978 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -138,6 +138,22 @@ lockless**. Local components skip the upstream release/changelog carryover and automatic commit steps. All lockless rendering runs on the host without mock; components using lock files retain their existing mock-based rendering. +Before any upstream checkout or component preparation, rendering checks the +entire project Git repository for staged changes, unstaged tracked changes, and +untracked files. Git-ignored files do not block rendering. If any selected +component is lockless and upstream, a dirty repository stops the whole invocation. +Commit or remove those changes first, or explicitly bypass this check: + +```sh +azldev comp render -p --allow-uncommitted +``` + +`--check-only` and `--no-git-commit` still require a clean repository unless +`--allow-uncommitted` is specified. The override bypasses only the cleanliness +check: carryover and upstream-change comparison still use committed history, +and failure recovery can discard uncommitted files in the component output +directory. Locked and local components do not trigger this check. + The renderer clones only the branch selected by the component's effective `spec.upstream-distro` (or the project default) into temporary storage. The resolved distro version's `dist-git-branch` supplies the branch name. The clone diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index a59aa469f..9cce6e923 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -54,6 +54,12 @@ preserved. When using a custom output directory (--output-dir), --force is required alongside --clean-stale as a safety measure. This flag is only valid with -a. +Before rendering any lockless upstream component, require a clean Git repository. +Uncommitted tracked changes and untracked files anywhere in the repository block +rendering; Git-ignored files are excluded. --allow-uncommitted bypasses this check. +--check-only and --no-git-commit do not bypass it. Locked and local components are +unaffected. + ``` azldev component render [flags] ``` @@ -75,6 +81,7 @@ azldev component render [flags] ``` -a, --all-components Include all components + --allow-uncommitted allow lockless upstream rendering with uncommitted or untracked files in the project repository --changelog-comment string comment passed verbatim to rpmdev-bumpspec -c and used as the Git commit subject for lockless upstream components (default: update ) --check-only render to a staging area and compare against the existing on-disk output without modifying the output directory. Exits 0 when nothing would change and 1 when any component would drift. With -a + --clean-stale, also fails on orphan rendered-spec directories. Intended for CI gates. --clean-stale prune rendered-spec directories that no longer correspond to a configured component (only with -a; requires -f with -o). Top-level non-component siblings are preserved. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 873c350a4..6408f4f23 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -180,6 +180,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "all prepared dist-git files except `.git`") assert.NotContains(t, doc, "host `spectool` filtering") assert.Contains(t, doc, "prepares the changelog before Release") + assert.Contains(t, doc, "Git-ignored files are excluded") + assert.Contains(t, doc, "`--allow-uncommitted`") assert.Contains(t, doc, "Version detection and carryover use the upstream spec before overlays") assert.Contains(t, doc, "Overlays can override the prepared release and changelog") assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") @@ -198,6 +200,9 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) require.NoError(t, err) assert.Contains(t, doc, "fully parsed committed TOML") + assert.Contains(t, doc, "Git-ignored files are excluded") + assert.Contains(t, doc, "`--allow-uncommitted`") + assert.Contains(t, doc, "`render --allow-uncommitted`") assert.Contains(t, doc, "automatically commits only that directory") assert.Contains(t, doc, "## RPMAUTOSPEC:") assert.Contains(t, doc, "leading integer of the committed,") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 36abaa4d3..fd599bd9f 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -41,6 +41,14 @@ generated upstream-commit TOML instead. All others use locks. then runs the refresh pipeline. Existing list contents and lock files are retained. Already-listed components are still refreshed; a refresh failure leaves the list updated. +Before preparing any selected lockless upstream component, rendering requires a +clean project Git repository: staged changes, unstaged tracked changes, and +untracked files anywhere in the repository block the whole invocation. +Git-ignored files are excluded. Use `--allow-uncommitted` for intentional dirty +renders; `--check-only` and `--no-git-commit` do not bypass this check. +The override does not change committed-history carryover or failure recovery. +Locked and local components do not trigger this check. + For listed upstream components, rendering prepares the changelog before Release, then writes macros, applies overlays, and updates `sources`. Version detection and carryover use the upstream spec before overlays. diff --git a/internal/app/azldev/agentskill/content/build-component.md.tmpl b/internal/app/azldev/agentskill/content/build-component.md.tmpl index 9375d5e22..42d00ccc8 100644 --- a/internal/app/azldev/agentskill/content/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/build-component.md.tmpl @@ -38,7 +38,13 @@ Prefer `comp render` for quick overlay verification; use `comp diff-sources` to exact overlay effect (it fetches sources once, applies overlays to a copy, and diffs the two trees). Builds can be slow — set generous timeouts. -Finalize with `azldev comp update -p ` before opening a PR (see the +For lockless upstream components, uncommitted tracked changes or untracked files +anywhere in the project repository require `comp render --allow-uncommitted`. +Git-ignored files are excluded; the override does not change committed-history +carryover or failure recovery. Commit inputs before the final render (see the +`azldev-refresh-upstream-commit` skill). + +For locked components, finalize with `azldev comp update -p ` before opening a PR (see the `azldev-update-component` skill). ## Debugging build failures diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index accac4347..400deeb40 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -30,7 +30,7 @@ Other configuration validation failures are still reported as warnings. | Adding a new upstream component | **Yes** — first, to generate its upstream commit TOML | | Finalizing overlay, build config, or metadata changes | No — these do not affect the resolved commit | | Changing source resolution (commit pin, upstream distro/version, or snapshot) | **Yes** — also mid-workflow (see below) | -| Iterating on overlays / build config / metadata | No — `render` alone is enough while iterating | +| Iterating on overlays / build config / metadata | No — use `render --allow-uncommitted` while inputs are uncommitted | | Just reading or building existing components | No | Refresh a single component with `-p `. Use `-a` (all components) only for @@ -52,6 +52,15 @@ Commit component inputs before rendering. For upstream components, rendering compares the fully parsed committed TOML at the last render commit and at HEAD; uncommitted generated pins do not count as a committed upstream change. +Before any upstream checkout or component preparation, rendering requires a +clean project Git repository: staged changes, unstaged tracked changes, and +untracked files anywhere in the repository block the whole invocation. +Git-ignored files are excluded. Use `--allow-uncommitted` for intentional dirty +renders; `--check-only` and `--no-git-commit` do not bypass this check. +The override does not change committed-history carryover or failure recovery, +which can discard uncommitted files in the component output directory. +Locked and local components do not trigger this check. + ```sh git add \ {{ .UpstreamCommitsDir }}/.toml diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index b671d4506..334e4efb3 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -38,6 +38,14 @@ images assemble RPMs into bootable artifacts. Existing list contents and lock files are retained. Already-listed components are still refreshed; a refresh failure leaves the list updated. +Before preparing any upstream component, rendering requires a clean project Git +repository: staged changes, unstaged tracked changes, and untracked files anywhere +in the repository block the whole invocation. Git-ignored files are excluded. +Use `--allow-uncommitted` for intentional dirty renders; `--check-only` and +`--no-git-commit` do not bypass this check. The override does not change +committed-history carryover or failure recovery. Local components do not trigger +this check. + For upstream components, rendering prepares the changelog before Release, then writes macros, applies overlays, and updates `sources`. Version detection and carryover use the upstream spec before overlays. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl index cac327d99..46bb3be38 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl @@ -41,6 +41,11 @@ two trees). Builds can be slow — set generous timeouts. Finalize with `azldev comp refresh-upstream-commit -p ` before opening a PR (see the `azldev-refresh-upstream-commit` skill). +For upstream components, uncommitted tracked changes or untracked files anywhere +in the project repository require `comp render --allow-uncommitted`. +Git-ignored files are excluded; the override does not change committed-history +carryover or failure recovery. Commit inputs before the final render. + ## Debugging build failures 1. **Render error mentioning a non-standard `Release` tag** — a release-calculation diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 678dcf65e..6009a54d9 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -41,6 +41,7 @@ type RenderOptions struct { NoChangeRebuild bool ChangelogComment string NoGitCommit bool + AllowUncommitted bool RPMDevBumpspec bool changelogCommentExplicit bool } @@ -49,6 +50,12 @@ func renderOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { parentCmd.AddCommand(NewRenderCmd()) } +const renderWorktreeHelp = `Before rendering any lockless upstream component, require a clean Git repository. +Uncommitted tracked changes and untracked files anywhere in the repository block +rendering; Git-ignored files are excluded. --allow-uncommitted bypasses this check. +--check-only and --no-git-commit do not bypass it. Locked and local components are +unaffected.` + // NewRenderCmd constructs a [cobra.Command] for the "component render" CLI subcommand. func NewRenderCmd() *cobra.Command { var options RenderOptions @@ -104,7 +111,7 @@ result table accurately reflects which components actually changed on disk. Top-level non-component siblings (e.g. a hand-placed README.md) are preserved. When using a custom output directory (--output-dir), --force is required alongside --clean-stale as a safety measure. This flag is only -valid with -a.`, +valid with -a.` + "\n\n" + renderWorktreeHelp, Example: ` # Render all components (output dir from config) azldev component render -a @@ -160,6 +167,8 @@ func addRenderOptions(cmd *cobra.Command, options *RenderOptions) { "for lockless upstream components (default: update )") cmd.Flags().BoolVar(&options.NoGitCommit, "no-git-commit", false, "leave lockless upstream rendered output uncommitted without staging it") + cmd.Flags().BoolVar(&options.AllowUncommitted, "allow-uncommitted", false, + "allow lockless upstream rendering with uncommitted or untracked files in the project repository") addRPMDevBumpspecFlag(cmd, &options.RPMDevBumpspec) // --check-only is a read-only diff against on-disk state; --fail-on-error @@ -209,14 +218,12 @@ func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, return nil, fmt.Errorf("failed to resolve components:\n%w", err) } - if comps.Len() == 0 { - return nil, errors.New("no components were selected; " + - "please use command-line options to indicate which components to render", - ) - } - componentList := comps.Components() + if err := validateRenderComponents(env, componentList, options.AllowUncommitted); err != nil { + return nil, err + } + mockProcessor, err := renderMockProcessor(env, componentList) if err != nil { return nil, err diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 20d4c1179..c46cd6279 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -61,6 +61,41 @@ type renderAutorelease interface { //nolint:gochecknoglobals // Replaced by isolated render tests to open an in-memory project repository. var openRenderRepo = openChangedRepo +func validateRenderComponents( + env *azldev.Env, comps []components.Component, allowUncommitted bool, +) error { + if len(comps) == 0 { + return errors.New("no components were selected; " + + "please use command-line options to indicate which components to render") + } + + if allowUncommitted { + return nil + } + + for _, comp := range comps { + if !env.ComponentWithoutLockfile(comp.GetName()) || + comp.GetConfig().Spec.SourceType != projectconfig.SpecSourceTypeUpstream { + continue + } + + status, err := git.RunInDir(env, env, env.ProjectDir(), + "status", "--porcelain=v1", "--untracked-files=all", "--ignore-submodules=none") + if err != nil { + return fmt.Errorf("checking project repository for uncommitted files:\n%w", err) + } + + if status != "" { + return errors.New("rendering will not be done due to uncommitted or untracked files " + + "in the project repository; use '--allow-uncommitted' to proceed") + } + + return nil + } + + return nil +} + func prepareLocklessRender( env *azldev.Env, comp components.Component, dir, outputDir string, editor spec.EditorMode, diff --git a/internal/app/azldev/cmds/component/render_modes_internal_test.go b/internal/app/azldev/cmds/component/render_modes_internal_test.go index 36dfef068..fe634cced 100644 --- a/internal/app/azldev/cmds/component/render_modes_internal_test.go +++ b/internal/app/azldev/cmds/component/render_modes_internal_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" @@ -23,6 +24,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" ) func TestReadLocklessComponents(t *testing.T) { @@ -88,6 +90,144 @@ func (p *renderModeObservingPreparer) PrepareSources( return p.err } +func TestRenderComponentsRequiresCleanLocklessUpstreamRepository(t *testing.T) { + statusErr := errors.New("git status failed") + + for _, testCase := range []struct { + name, status, wantError string + allow, checkOnly, noGitCommit, mixed bool + statusErr error + }{ + {name: "clean"}, + {name: "unstaged tracked", status: " M tracked\n", wantError: "uncommitted or untracked files"}, + {name: "staged tracked", status: "M tracked\n", wantError: "uncommitted or untracked files"}, + {name: "untracked", status: "?? scratch\n", wantError: "uncommitted or untracked files"}, + {name: "explicit override", status: "?? scratch\n", allow: true}, + { + name: "check-only still checks", status: " M tracked\n", + checkOnly: true, wantError: "uncommitted or untracked files", + }, + { + name: "no-git-commit still checks", status: " M tracked\n", + noGitCommit: true, wantError: "uncommitted or untracked files", + }, + { + name: "mixed selection stops before preparation", status: "?? scratch\n", + mixed: true, wantError: "uncommitted or untracked files", + }, + {name: "status failure", statusErr: statusErr, wantError: "checking project repository"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + if testCase.mixed { + env = testutils.NewTestEnv(t) + env.AddUpstreamComponent(t, "bash") + env.Config.LocklessComponents = map[string]bool{"curl": true} + } + + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeUpstream, UpstreamCommit: "configured-commit", + }, + } + queried := false + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + queried = true + + assert.Equal(t, []string{ + "git", "-C", "/project", "status", "--porcelain=v1", + "--untracked-files=all", "--ignore-submodules=none", + }, cmd.Args) + + return testCase.status, testCase.statusErr + } + started := false + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, _ opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + started = true + + return &renderModeObservingPreparer{ + observe: func(_ components.Component) {}, + err: errors.New("stop after preflight"), + }, nil + } + + t.Cleanup(func() { newSourcePreparer = original }) + + results, err := RenderComponents(env.Env, &RenderOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + OutputDir: "/project/SPECS", + OutputDirExplicit: true, + AllowUncommitted: testCase.allow, + CheckOnly: testCase.checkOnly, + NoGitCommit: testCase.noGitCommit, + }) + assert.Equal(t, !testCase.allow, queried) + + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + assert.Empty(t, results) + assert.False(t, started, "refuse before fetching any selected component") + + if testCase.statusErr != nil { + require.ErrorIs(t, err, statusErr) + } else { + assert.Contains(t, err.Error(), "--allow-uncommitted") + } + } else { + require.NoError(t, err) + assert.True(t, started) + require.Len(t, results, 1) + assert.Contains(t, results[0].Error, "stop after preflight") + } + }) + } +} + +func TestLocklessRenderWorktreeCheckScope(t *testing.T) { + for _, testCase := range []struct { + name string + source projectconfig.SpecSourceType + lockless bool + checked bool + }{ + {name: "locked upstream", source: projectconfig.SpecSourceTypeUpstream}, + {name: "locked local", source: projectconfig.SpecSourceTypeLocal}, + {name: "lockless local", source: projectconfig.SpecSourceTypeLocal, lockless: true}, + {name: "lockless upstream", source: projectconfig.SpecSourceTypeUpstream, lockless: true, checked: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": testCase.lockless, "rust-asyncband": true} + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: testCase.source}, + }) + + queried := false + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + queried = true + + return "?? scratch\n", nil + } + err := validateRenderComponents(env.Env, []components.Component{comp}, false) + + assert.Equal(t, testCase.checked, queried) + + if testCase.checked { + require.ErrorContains(t, err, "uncommitted or untracked files") + } else { + require.NoError(t, err) + } + }) + } +} + func TestLocklessRenderPreservesAllFilesWithoutMock(t *testing.T) { env := testutils.NewTestEnvWithoutLockfile(t) version := env.Config.Distros["test-distro"].Versions["1.0"] diff --git a/internal/app/azldev/cmds/component/render_test.go b/internal/app/azldev/cmds/component/render_test.go index 2367053be..0930d293e 100644 --- a/internal/app/azldev/cmds/component/render_test.go +++ b/internal/app/azldev/cmds/component/render_test.go @@ -4,6 +4,8 @@ package component_test import ( + "os/exec" + "slices" "testing" componentcmds "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" @@ -63,6 +65,12 @@ func TestNewRenderCmd_Flags(t *testing.T) { assert.Equal(t, "Fix curl patches", commentFlag.Value.String()) assert.Equal(t, "true", noGitCommitFlag.Value.String()) + allowUncommittedFlag := cmd.Flags().Lookup("allow-uncommitted") + require.NotNil(t, allowUncommittedFlag) + assert.Equal(t, "false", allowUncommittedFlag.DefValue) + require.NoError(t, cmd.ParseFlags([]string{"--allow-uncommitted"})) + assert.Equal(t, "true", allowUncommittedFlag.Value.String()) + rpmDevBumpspecFlag := cmd.Flags().Lookup("rpmdev-bumpspec") require.NotNil(t, rpmDevBumpspecFlag) assert.Equal(t, "false", rpmDevBumpspecFlag.DefValue) @@ -87,6 +95,54 @@ func TestRenderCmd_NoComponents(t *testing.T) { assert.Contains(t, err.Error(), "component not found") } +func TestRenderCmd_AllowUncommitted(t *testing.T) { + for _, testCase := range []struct { + name string + allow bool + }{ + {name: "dirty repository refused"}, + {name: "override skips status", allow: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.AddUpstreamComponent(t, "curl") + env.Config.LocklessComponents = map[string]bool{"curl": true} + config := env.Config.Components["curl"] + config.Spec.UpstreamCommit = "configured-commit" + env.Config.Components["curl"] = config + queried := false + original := env.CmdFactory.RunAndGetOutputHandler + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if slices.Contains(cmd.Args, "--porcelain=v1") { + queried = true + + return "?? scratch\n", nil + } + + return original(cmd) + } + cmd := componentcmds.NewRenderCmd() + + args := []string{"-p", "curl", "-o", "SPECS", "--fail-on-error"} + if testCase.allow { + args = append(args, "--allow-uncommitted") + } + + cmd.SetArgs(args) + err := cmd.ExecuteContext(env.Env) + + assert.Equal(t, !testCase.allow, queried) + require.Error(t, err) + + if !testCase.allow { + assert.Contains(t, err.Error(), "uncommitted or untracked files") + } else { + assert.NotContains(t, err.Error(), "uncommitted or untracked files") + } + }) + } +} + func TestRenderCmd_ChangelogCommentRequiresValue(t *testing.T) { testEnv := testutils.NewTestEnv(t) cmd := componentcmds.NewRenderCmd() From 9a4c90441b02782d4b3a71c92aac627647e4a01f Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 20:49:08 +0000 Subject: [PATCH 32/70] feat(component): include pending inputs in first render commit Stage all uncommitted repository changes outside the rendered output directory and include them with the first successfully committed lockless upstream component. Later commits retain component-only scope. Use literal pathspecs for input filenames and NUL-delimited status data so leading whitespace, pathspec metacharacters, and deleted files remain safe. Exclude other rendered paths even when already staged. Empty or failed commit attempts do not consume first-commit input inclusion. Update user guidance, CLI documentation, emitted skills, and focused commit-sequencing tests. Recovery still resets only component paths; outside inputs may remain staged if committing fails. Validation: mage fix all, mage unit, mage check all, and mage docs passed. A real two-component render using local upstream fixtures verified that the first commit contains outside tracked, staged, untracked, and deleted inputs plus curl; the second contains only vim. Unrelated staged and untracked rendered paths remained pending. CLI help and full agent installation checks passed. Peak validation memory was 2 GiB. mage all and mage scenarioUpdate remain Docker/SELinux-blocked. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 11 +- .../reference/cli/azldev_component_render.md | 6 +- .../app/azldev/agentskill/agentskill_test.go | 6 +- .../azldev/agentskill/content/azldev.md.tmpl | 7 +- .../agentskill/content/comp-toml.md.tmpl | 2 + .../content/refresh-upstream-commit.md.tmpl | 9 +- .../content/withoutlockfile/azldev.md.tmpl | 6 +- .../content/withoutlockfile/comp-toml.md.tmpl | 2 + internal/app/azldev/cmds/component/render.go | 6 +- .../azldev/cmds/component/render_lockless.go | 63 +++++++-- .../render_lockless_internal_test.go | 123 +++++++++++++++++- 11 files changed, 211 insertions(+), 30 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 18b538978..09aaaf614 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -297,9 +297,14 @@ existing output directory and moves all prepared entries except `.git` into its replacement. Files found only in the old output are not retained. Moves use filesystem renames, so the project work directory and rendered output must be on the same filesystem. -The renderer then commits only the component output directory. The subject is +The renderer then commits the component output directory. The subject is the same changelog comment, defaulting to `update `; the body -lists upstream changes, if any. Unrelated staged files are not committed. +lists upstream changes, if any. The first successful automatic commit also +includes all staged changes, unstaged tracked changes, and non-ignored untracked +files outside `rendered-specs-dir` (or the active `--output-dir`). Other component +directories and unrelated changes within the rendered output directory are +excluded from that commit, even if already staged. Later component commits +include only their own dist-git output. Unchanged Git content does not produce an empty commit. Pass `--no-git-commit` to skip automatic staging and committing while still rendering and bumping the spec. The rendered output remains available for review @@ -319,6 +324,8 @@ contents from `HEAD`; otherwise, the directory remains absent. This recovery does not preserve uncommitted changes or untracked files in the replaced component directory, and uses no backup or index snapshot. Recovery errors are reported with the original render failure. +Files outside the rendered output directory may remain staged if automatic +committing fails; component recovery does not reset those paths. `--check-only` failures leave output untouched and fail the check. Locked rendering retains its failure marker behavior. diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 9cce6e923..833dbe868 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -31,9 +31,9 @@ upstream spec and last committed render before writing macros, applying overlays and updating sources. Overlays can override the prepared release and changelog. Compare output; unchanged renders require --no-change-rebuild to record a timestamp. Then run rpmdev-bumpspec -c with --changelog-comment (default: "update "). -Commit only that directory with the same subject and upstream changes as the body. ---no-git-commit skips staging and committing. Both flags affect only lockless upstream -components; unrelated staged files are excluded. +Commit that directory with the same subject and upstream changes as the body. +The first commit also includes all pending changes outside the rendered output directory. +Later commits are component-only. --no-git-commit skips staging and committing. Output must be inside the project repository; configure Git author identity and install rpmdev-bumpspec. Commit updated upstream-commit TOML before rendering. --check-only stops before bumping, replacing output, or committing. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 6408f4f23..ae8dae49b 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -182,6 +182,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "prepares the changelog before Release") assert.Contains(t, doc, "Git-ignored files are excluded") assert.Contains(t, doc, "`--allow-uncommitted`") + assert.Contains(t, doc, "first commit also includes") + assert.Contains(t, doc, "Other rendered paths are excluded even if staged") assert.Contains(t, doc, "Version detection and carryover use the upstream spec before overlays") assert.Contains(t, doc, "Overlays can override the prepared release and changelog") assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") @@ -203,7 +205,9 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "Git-ignored files are excluded") assert.Contains(t, doc, "`--allow-uncommitted`") assert.Contains(t, doc, "`render --allow-uncommitted`") - assert.Contains(t, doc, "automatically commits only that directory") + assert.Contains(t, doc, "automatically commits that directory") + assert.Contains(t, doc, "first commit also includes") + assert.Contains(t, doc, "Other rendered paths are excluded even if staged") assert.Contains(t, doc, "## RPMAUTOSPEC:") assert.Contains(t, doc, "leading integer of the committed,") assert.Contains(t, doc, "an omitted `-b` stays omitted") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index fd599bd9f..4a1325123 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -68,8 +68,11 @@ Recovery discards uncommitted changes in that directory and keeps no backups. It never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. It retains all prepared dist-git files except `.git`, without file filtering or `spectool` queries, and replaces and -automatically commits only that component's output directory; unrelated staged -files are excluded. `--changelog-comment` supplies the exact `rpmdev-bumpspec -c` +automatically commits that component's output directory. The first commit also includes +all staged changes, unstaged tracked changes, and non-ignored untracked files outside +the rendered output directory. Other rendered paths are excluded even if staged; +later commits include only their component's dist-git changes. +`--changelog-comment` supplies the exact `rpmdev-bumpspec -c` value and commit subject, defaulting to `update `. `--no-git-commit` skips automatic staging and committing, but still renders and bumps. Upstream changes remain in the Git commit body, not the changelog comment. diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 71672d04b..7b11ffac9 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -96,6 +96,8 @@ used outside the lockless upstream render workflow. That workflow always runs host `rpmdev-bumpspec -c` before existing processing, accepts its `%autorelease` no-op, and automatically commits the component output. Lockless rendering skips synthetic history and its legacy release adjustment. Commit input TOML first. +The first automatic commit also includes pending repository changes outside the +rendered output directory; later commits contain only their component output. ## Render configuration diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 400deeb40..fd2facd67 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -154,11 +154,16 @@ committed lock file can supply the old pin if that commit's top-level render still requires a configured pin. Rendering then replaces `{{ .RenderedSpecsDir }}///` and -automatically commits only that directory. The subject is the same changelog +automatically commits that directory. The first commit also includes all staged +changes, unstaged tracked changes, and non-ignored untracked files outside the +rendered output directory. Other rendered paths are excluded even if staged; +later commits include only their component's dist-git changes. +The subject is the same changelog comment, defaulting to `update `, and the body contains upstream changes, if any. `--no-git-commit` skips automatic staging and committing, but still renders and bumps. -It never includes unrelated staged files or creates empty commits. Configure +It never creates empty commits. Files outside rendered output may remain staged +if committing fails; component recovery does not reset them. Configure Git author identity and install host `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. `--check-only` never replaces output, creates rebuild timestamps, or commits. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 334e4efb3..cf4e59473 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -61,7 +61,11 @@ Lockless failures never write `RENDER_FAILED`. Before replacement, output is untouched; failures after replacement starts clear the component directory and use Git to restore committed output, or leave no directory if none existed. Recovery discards uncommitted changes in that directory and keeps no backups. -It replaces and automatically commits only the component output directory. +It replaces and automatically commits the component output directory. +The first commit also includes all staged changes, unstaged tracked changes, +and non-ignored untracked files outside the rendered output directory. +Other rendered paths are excluded even if staged; later commits include only +their component's dist-git changes. `--changelog-comment` supplies the exact `rpmdev-bumpspec -c` value and commit subject, defaulting to `update `. `--no-git-commit` skips automatic staging and committing, but still renders and bumps. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 37087bddf..6436007ab 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -95,6 +95,8 @@ or `rpmautospec process-distgit`; RPM macros remain unexpanded and the adjustment are skipped during rendering. `--check-only` never creates project commits. Local components skip upstream carryover and automatic commits. +The first automatic commit also includes pending repository changes outside the +rendered output directory; later commits contain only their component output. azldev reads `lockless-components.txt` beside the top-level `azldev.toml`. Listed component names use the lockless workflow for every command; diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 6009a54d9..05223da02 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -90,9 +90,9 @@ upstream spec and last committed render before writing macros, applying overlays and updating sources. Overlays can override the prepared release and changelog. Compare output; unchanged renders require --no-change-rebuild to record a timestamp. Then run rpmdev-bumpspec -c with --changelog-comment (default: "update "). -Commit only that directory with the same subject and upstream changes as the body. ---no-git-commit skips staging and committing. Both flags affect only lockless upstream -components; unrelated staged files are excluded. +Commit that directory with the same subject and upstream changes as the body. +The first commit also includes all pending changes outside the rendered output directory. +Later commits are component-only. --no-git-commit skips staging and committing. Output must be inside the project repository; configure Git author identity and install rpmdev-bumpspec. Commit updated upstream-commit TOML before rendering. --check-only stops before bumping, replacing output, or committing. diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index c46cd6279..8ce1eebb7 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -864,13 +864,18 @@ func commitLocklessRenders( } // Serialize writes to the project index. + includeInputs := true + for _, prep := range prepared { result := results[prep.index] if prep.update != nil && !prep.update.noGitCommit && result.Status == renderStatusOK { - if err := commitLocklessRender(env, prep.update); err != nil { + committed, err := commitLocklessRender(env, prep.update, includeInputs) + if err != nil { result.Status = renderStatusError result.Error = err.Error() + } else if committed { + includeInputs = false } } @@ -922,18 +927,30 @@ func restoreLocklessOutput(env *azldev.Env, prep *preparedComponent) error { return nil } -func commitLocklessRender(env *azldev.Env, update *locklessRenderUpdate) error { +func commitLocklessRender(env *azldev.Env, update *locklessRenderUpdate, includeInputs bool) (bool, error) { + paths := []string{update.path} + if includeInputs { + inputs, err := stageLocklessRenderInputs(env, update) + if err != nil { + return false, err + } + + paths = append(paths, inputs...) + } + if _, err := git.RunInDir(env, env, update.repoRoot, "add", "--all", "--", update.path); err != nil { - return fmt.Errorf("staging rendered component:\n%w", err) + return false, fmt.Errorf("staging rendered component:\n%w", err) } - diff, err := git.RunInDir(env, env, update.repoRoot, "diff", "--cached", "--name-only", "--", update.path) + diffArgs := append([]string{"diff", "--cached", "--name-only", "--"}, paths...) + + diff, err := git.RunInDir(env, env, update.repoRoot, diffArgs...) if err != nil { - return fmt.Errorf("checking staged render:\n%w", err) + return false, fmt.Errorf("checking staged render:\n%w", err) } if diff == "" { - return nil + return false, nil } args := []string{"commit", "--only", "-m", update.subject} @@ -941,10 +958,38 @@ func commitLocklessRender(env *azldev.Env, update *locklessRenderUpdate) error { args = append(args, "-m", update.changes) } - args = append(args, "--", update.path) + args = append(args, "--") + + args = append(args, paths...) if _, err := git.RunInDir(env, env, update.repoRoot, args...); err != nil { - return fmt.Errorf("committing rendered component:\n%w", err) + return false, fmt.Errorf("committing rendered component:\n%w", err) } - return nil + return true, nil +} + +func stageLocklessRenderInputs(env *azldev.Env, update *locklessRenderUpdate) ([]string, error) { + renderedPath := filepath.ToSlash(filepath.Dir(filepath.Dir(update.path))) + exclusion := ":(top,exclude,literal)" + renderedPath + + if _, err := git.RunInDir(env, env, update.repoRoot, "add", "--all", "--", ".", exclusion); err != nil { + return nil, fmt.Errorf("staging uncommitted files outside rendered specs:\n%w", err) + } + + status, err := git.RunInDir(env, env, update.repoRoot, + "diff", "--cached", "--name-status", "--no-renames", "-z", "--", ".", exclusion) + if err != nil { + return nil, fmt.Errorf("checking staged files outside rendered specs:\n%w", err) + } + + // With '--no-renames', NUL-delimited name/status output contains status/path pairs. + fields := strings.Split(status, "\x00") + + var paths []string + + for index := 1; index < len(fields); index += 2 { + paths = append(paths, ":(top,literal)"+fields[index]) + } + + return paths, nil } diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 4d5f482f9..fc50bd814 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -9,6 +9,7 @@ import ( "fmt" "os/exec" "path/filepath" + "slices" "strconv" "strings" "testing" @@ -1190,6 +1191,10 @@ upstream-commit = "aaaaaaa" assert.False(t, testCase.noGitCommit, "no Git commands should run for --no-git-commit") if cmd.Args[3] == renderTestDiffCommand { + if slices.Contains(cmd.Args, "--name-status") { + return "", nil + } + return renderTestSpecDiff, nil } @@ -1208,7 +1213,9 @@ upstream-commit = "aaaaaaa" if testCase.noGitCommit { assert.Empty(t, gitCommands) } else { - assert.Equal(t, []string{"add", renderTestDiffCommand, "commit"}, gitCommands) + assert.Equal(t, []string{ + "add", renderTestDiffCommand, "add", renderTestDiffCommand, "commit", + }, gitCommands) } assert.Equal(t, renderStatusOK, results[0].Status) @@ -1478,9 +1485,12 @@ func TestCommitLocklessRenders_SkipsFailedComponents(t *testing.T) { env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { commands = append(commands, cmd.Args) - assert.Equal(t, "SPECS/v/vim", cmd.Args[len(cmd.Args)-1]) - if cmd.Args[3] == "diff" { + if cmd.Args[3] == renderTestDiffCommand { + if slices.Contains(cmd.Args, "--name-status") { + return "M\x00config.toml\x00", nil + } + return "SPECS/v/vim/vim.spec\n", nil } @@ -1501,8 +1511,105 @@ func TestCommitLocklessRenders_SkipsFailedComponents(t *testing.T) { commitLocklessRenders(env.Env, prepared, results, false) assert.Equal(t, "render failed", results[1].Error) assert.Equal(t, renderStatusOK, results[2].Status) - require.Len(t, commands, 3) - assert.Equal(t, "commit", commands[2][3]) + require.Len(t, commands, 5) + assert.Equal(t, []string{ + "git", "-C", "/project", "add", "--all", "--", ".", ":(top,exclude,literal)SPECS", + }, commands[0]) + assert.Equal(t, []string{ + "git", "-C", "/project", "commit", "--only", "-m", "updated vim", + "--", "SPECS/v/vim", ":(top,literal)config.toml", + }, commands[4]) +} + +func TestCommitLocklessRenders_InputsOnlyInFirstCommit(t *testing.T) { + for _, testCase := range []struct { + name string + empty, fail bool + wantInputQueries int + }{ + {name: "successful first commit", wantInputQueries: 1}, + {name: "empty first attempt", empty: true, wantInputQueries: 2}, + {name: "failed first commit", fail: true, wantInputQueries: 2}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + inputQueries := 0 + + var commits [][]string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + switch cmd.Args[3] { + case "add": + if cmd.Args[len(cmd.Args)-1] == ":(top,exclude,literal)nested/rendered" { + assert.Equal(t, ".", cmd.Args[len(cmd.Args)-2]) + } + case renderTestDiffCommand: + if slices.Contains(cmd.Args, "--name-status") { + inputQueries++ + + assert.Contains(t, cmd.Args, ":(top,exclude,literal)nested/rendered") + assert.Contains(t, cmd.Args, "--no-renames") + assert.Contains(t, cmd.Args, "-z") + + if testCase.empty && inputQueries == 1 { + return "", nil + } + + return "M\x00config.toml\x00A\x00 scratch[1].txt\x00D\x00removed.txt\x00", nil + } + + if testCase.empty && slices.Contains(cmd.Args, "nested/rendered/c/curl") { + return "", nil + } + + return "changed\n", nil + case "commit": + commits = append(commits, cmd.Args) + if testCase.fail && slices.Contains(cmd.Args, "update curl") { + return "", errors.New("first commit failed") + } + default: + t.Fatalf("unexpected Git command: %v", cmd.Args) + } + + return "", nil + } + prepared := []*preparedComponent{ + {index: 0, update: &locklessRenderUpdate{ + repoRoot: "/project", path: "nested/rendered/c/curl", subject: "update curl", + }}, + {index: 1, update: &locklessRenderUpdate{ + repoRoot: "/project", path: "nested/rendered/v/vim", subject: "update vim", + }}, + } + results := []*RenderResult{{Status: renderStatusOK}, {Status: renderStatusOK}} + + commitLocklessRenders(env.Env, prepared, results, false) + + assert.Equal(t, testCase.wantInputQueries, inputQueries) + require.NotEmpty(t, commits) + firstCommit := commits[0] + assert.Contains(t, firstCommit, ":(top,literal)config.toml") + assert.Contains(t, firstCommit, ":(top,literal) scratch[1].txt") + assert.Contains(t, firstCommit, ":(top,literal)removed.txt") + + if !testCase.empty { + assert.Contains(t, firstCommit, "nested/rendered/c/curl") + assert.NotContains(t, firstCommit, "nested/rendered/v/vim") + } + + lastCommit := commits[len(commits)-1] + assert.Contains(t, lastCommit, "nested/rendered/v/vim") + assert.NotContains(t, lastCommit, "nested/rendered/c/curl") + assert.Equal(t, renderStatusOK, results[1].Status) + + if testCase.fail || testCase.empty { + assert.Contains(t, lastCommit, ":(top,literal)config.toml") + } else { + assert.NotContains(t, lastCommit, ":(top,literal)config.toml") + } + }) + } } func TestCommitLocklessRender_PathOnlyAndNoEmptyCommits(t *testing.T) { @@ -1514,7 +1621,7 @@ func TestCommitLocklessRender_PathOnlyAndNoEmptyCommits(t *testing.T) { env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { commands = append(commands, cmd.Args) - if cmd.Args[3] == "diff" && changed { + if cmd.Args[3] == renderTestDiffCommand && changed { return renderTestSpecDiff, nil } @@ -1523,7 +1630,9 @@ func TestCommitLocklessRender_PathOnlyAndNoEmptyCommits(t *testing.T) { update := &locklessRenderUpdate{ repoRoot: "/project", path: "SPECS/c/curl", subject: "updated curl", changes: "abc123 Fix", } - require.NoError(t, commitLocklessRender(env.Env, update)) + committed, err := commitLocklessRender(env.Env, update, false) + require.NoError(t, err) + assert.Equal(t, changed, committed) assert.Equal(t, []string{"git", "-C", "/project", "add", "--all", "--", update.path}, commands[0]) if changed { From cc582cfd4fd2170dc1df0312a6cbabb4e7ae379c Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Thu, 5 Nov 2026 21:31:18 +0000 Subject: [PATCH 33/70] feat(component): prompt for lockless render messages Remove the implicit changelog comment. For non-autorelease specs, obtain missing text through EDITOR or a local editor before rpmdev-bumpspec. Pass the edited content unchanged to bumpspec and reuse it for the Git commit message. Abort on empty edited text or editor failure. Skip rpmdev-bumpspec entirely for autorelease specs. When no comment was provided, let Git prompt for a message using an upstream-change template. Keep explicit messages, check-only, dry-run, and no-git-commit behavior consistent. Serialize finishing when a message may need interactive input. Update regression tests, user documentation, generated CLI reference, and emitted agent skills. Validation: mage unit, mage check all, and mage docs passed. Real CLI smoke tests verified edited-message reuse, local editor fallback, provided messages, native Git prompting, and autorelease no-commit behavior. Peak validation memory was approximately 2.2 GiB. mage all and mage scenarioUpdate remain Docker/SELinux-blocked. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 27 ++- .../reference/cli/azldev_component_render.md | 6 +- .../app/azldev/agentskill/agentskill_test.go | 6 +- .../azldev/agentskill/content/azldev.md.tmpl | 11 +- .../agentskill/content/comp-toml.md.tmpl | 6 +- .../content/refresh-upstream-commit.md.tmpl | 11 +- .../content/withoutlockfile/azldev.md.tmpl | 11 +- .../content/withoutlockfile/comp-toml.md.tmpl | 6 +- internal/app/azldev/cmds/component/render.go | 16 +- .../cmds/component/render_internal_test.go | 11 +- .../azldev/cmds/component/render_lockless.go | 49 +++-- .../render_lockless_internal_test.go | 51 +++-- .../azldev/cmds/component/render_message.go | 157 ++++++++++++++ .../component/render_message_internal_test.go | 202 ++++++++++++++++++ .../component/render_output_internal_test.go | 4 +- 15 files changed, 494 insertions(+), 80 deletions(-) create mode 100644 internal/app/azldev/cmds/component/render_message.go create mode 100644 internal/app/azldev/cmds/component/render_message_internal_test.go diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 09aaaf614..331298408 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -266,10 +266,22 @@ The root marker is ignored in both the working upstream checkout and existing output, even if its contents differ. Nested files with the same name are compared. Changed renders, or explicitly requested no-change rebuilds, then invoke host -`rpmdev-bumpspec -c` with the exact `--changelog-comment` value. If omitted, the -comment defaults to `update ` for each rendered component. -Upstream changes are not appended to this comment. -The tool's no-op for `%autorelease` specs is accepted. +`rpmdev-bumpspec -c` **only for non-autorelease specs**, with the exact +`--changelog-comment` value. There is no default comment. If the flag is omitted, +rendering opens `$EDITOR` (including configured arguments), or a local editor +(`editor`, `sensible-editor`, `vi`, or `nano`), to obtain the entry. Save and close +the editor to continue. Its text is passed unchanged to `rpmdev-bumpspec -c` and +reused as the Git commit message. Empty edited text or editor failure aborts +rendering before output replacement. Missing-comment finishing is serialized +to prevent simultaneous editors. + +Autorelease specs skip `rpmdev-bumpspec` entirely. With `--changelog-comment`, +Git uses the supplied message; without it, Git opens its own configured editor +to obtain the commit message, using a template containing upstream changes. +`--no-git-commit` skips that Git prompt, but non-autorelease specs still need a +changelog entry. `--check-only` and dry runs never open editors. +For unattended rendering, provide `--changelog-comment`. +Upstream changes are not appended to the changelog entry. ```bash azldev comp render -p curl --changelog-comment "Fix curl patches" @@ -298,8 +310,9 @@ its replacement. Files found only in the old output are not retained. Moves use filesystem renames, so the project work directory and rendered output must be on the same filesystem. The renderer then commits the component output directory. The subject is -the same changelog comment, defaulting to `update `; the body -lists upstream changes, if any. The first successful automatic commit also +the supplied or edited changelog comment; autorelease without a comment uses +Git's interactive message editor. The body includes upstream changes, if any. +The first successful automatic commit also includes all staged changes, unstaged tracked changes, and non-ignored untracked files outside `rendered-specs-dir` (or the active `--output-dir`). Other component directories and unrelated changes within the rendered output directory are @@ -307,7 +320,7 @@ excluded from that commit, even if already staged. Later component commits include only their own dist-git output. Unchanged Git content does not produce an empty commit. Pass `--no-git-commit` to skip automatic staging and committing while still -rendering and bumping the spec. The rendered output remains available for review +rendering and bumping non-autorelease specs. The rendered output remains available for review and manual commit. Both flags affect only lockless upstream components; locked and local rendering retain their existing behavior. diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 833dbe868..2501c03ff 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -30,8 +30,8 @@ For lockless upstream components, prepare the changelog, then Release, using the upstream spec and last committed render before writing macros, applying overlays, and updating sources. Overlays can override the prepared release and changelog. Compare output; unchanged renders require --no-change-rebuild to record a timestamp. -Then run rpmdev-bumpspec -c with --changelog-comment (default: "update "). -Commit that directory with the same subject and upstream changes as the body. +Non-autorelease specs use rpmdev-bumpspec -c with --changelog-comment, or edited text. +Autorelease skips bumpspec; without --changelog-comment, Git prompts for a message. The first commit also includes all pending changes outside the rendered output directory. Later commits are component-only. --no-git-commit skips staging and committing. Output must be inside the project repository; configure Git author identity and @@ -82,7 +82,7 @@ azldev component render [flags] ``` -a, --all-components Include all components --allow-uncommitted allow lockless upstream rendering with uncommitted or untracked files in the project repository - --changelog-comment string comment passed verbatim to rpmdev-bumpspec -c and used as the Git commit subject for lockless upstream components (default: update ) + --changelog-comment string comment passed verbatim to rpmdev-bumpspec -c and used as the Git commit message for lockless upstream components (if omitted, prompt in an editor) --check-only render to a staging area and compare against the existing on-disk output without modifying the output directory. Exits 0 when nothing would change and 1 when any component would drift. With -a + --clean-stale, also fails on orphan rendered-spec directories. Intended for CI gates. --clean-stale prune rendered-spec directories that no longer correspond to a configured component (only with -a; requires -f with -o). Top-level non-component siblings are preserved. -p, --component stringArray Component name pattern diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index ae8dae49b..a8dada91e 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -193,7 +193,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") assert.Contains(t, doc, "`--changelog-comment`") - assert.Contains(t, doc, "`update `") + assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") + assert.Contains(t, doc, "`$EDITOR` or a local editor") assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") assert.Contains(t, doc, "`azldev comp make-lockless`") assert.Contains(t, doc, "spaces and tabs after `Release:`") @@ -235,7 +236,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") assert.Contains(t, doc, "`--changelog-comment`") - assert.Contains(t, doc, "`update `") + assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") + assert.Contains(t, doc, "`$EDITOR` or a local editor") assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") assert.Contains(t, doc, "`azldev comp make-lockless`") assert.Contains(t, doc, "spaces and tabs after `Release:`") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 4a1325123..17536c138 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -53,7 +53,7 @@ For listed upstream components, rendering prepares the changelog before Release, then writes macros, applies overlays, and updates `sources`. Version detection and carryover use the upstream spec before overlays. Overlays can override the prepared release and changelog. -Rendering then runs host `rpmdev-bumpspec`, without synthetic history or its legacy +Non-autorelease rendering then runs host `rpmdev-bumpspec`, without synthetic history or its legacy release adjustment. Lockless rendering preserves the existing spaces and tabs after `Release:` when editing its value, and compares output before the final release bump; `--check-only` stops before that bump. @@ -73,8 +73,13 @@ all staged changes, unstaged tracked changes, and non-ignored untracked files ou the rendered output directory. Other rendered paths are excluded even if staged; later commits include only their component's dist-git changes. `--changelog-comment` supplies the exact `rpmdev-bumpspec -c` -value and commit subject, defaulting to `update `. -`--no-git-commit` skips automatic staging and committing, but still renders and bumps. +value and commit message, with no default. If omitted, `$EDITOR` or a local editor +collects the entry; the exact edited text is reused for the Git message. +Autorelease skips `rpmdev-bumpspec`; without a supplied comment, Git prompts +in its own editor for a message, with upstream changes in the template. +Provide `--changelog-comment` for unattended rendering. +`--no-git-commit` skips automatic staging and committing, but still renders and +bumps non-autorelease specs. Upstream changes remain in the Git commit body, not the changelog comment. Commit refreshed TOML before rendering. Configure Git author identity and install `rpmdevtools`. `--check-only` never creates project commits. diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 7b11ffac9..e51203819 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -93,8 +93,10 @@ identity/comment/datestamp and strict source-EVR transaction checks; locks and synthetic history still determine the number and order of bumps. Without the flag, no bumpspec runtime discovery occurs and the legacy static calculation is used outside the lockless upstream render workflow. That workflow always runs -host `rpmdev-bumpspec -c` before existing processing, accepts its `%autorelease` -no-op, and automatically commits the component output. Lockless rendering skips +host `rpmdev-bumpspec -c` only for non-autorelease specs and automatically commits +the component output. Missing `--changelog-comment` opens `$EDITOR` or a local +editor for non-autorelease, or Git's own editor for an autorelease commit. +Lockless rendering skips synthetic history and its legacy release adjustment. Commit input TOML first. The first automatic commit also includes pending repository changes outside the rendered output directory; later commits contain only their component output. diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index fd2facd67..42f4b8f9d 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -158,10 +158,13 @@ automatically commits that directory. The first commit also includes all staged changes, unstaged tracked changes, and non-ignored untracked files outside the rendered output directory. Other rendered paths are excluded even if staged; later commits include only their component's dist-git changes. -The subject is the same changelog -comment, defaulting to `update `, and the body contains upstream -changes, if any. -`--no-git-commit` skips automatic staging and committing, but still renders and bumps. +The message is the supplied changelog comment or, for non-autorelease specs, +the entry collected through `$EDITOR` or a local editor. There is no default. +Autorelease skips `rpmdev-bumpspec`; without a supplied comment, Git prompts +in its own editor for a message. The body/template contains upstream changes. +Provide `--changelog-comment` for unattended rendering. +`--no-git-commit` skips automatic staging and committing, but still renders and +bumps non-autorelease specs. It never creates empty commits. Files outside rendered output may remain staged if committing fails; component recovery does not reset them. Configure Git author identity and install host `rpmdevtools`, `rpm-build`, and `rpmautospec` diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index cf4e59473..584455236 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -50,7 +50,7 @@ For upstream components, rendering prepares the changelog before Release, then writes macros, applies overlays, and updates `sources`. Version detection and carryover use the upstream spec before overlays. Overlays can override the prepared release and changelog. -Rendering then runs host `rpmdev-bumpspec`. +Non-autorelease rendering then runs host `rpmdev-bumpspec`. Release edits preserve the existing spaces and tabs after `Release:`. Rendering compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors @@ -67,8 +67,13 @@ and non-ignored untracked files outside the rendered output directory. Other rendered paths are excluded even if staged; later commits include only their component's dist-git changes. `--changelog-comment` supplies the exact `rpmdev-bumpspec -c` value and commit -subject, defaulting to `update `. -`--no-git-commit` skips automatic staging and committing, but still renders and bumps. +message, with no default. If omitted, `$EDITOR` or a local editor collects the +entry; the exact edited text is reused for the Git message. +Autorelease skips `rpmdev-bumpspec`; without a supplied comment, Git prompts +in its own editor for a message, with upstream changes in the template. +Provide `--changelog-comment` for unattended rendering. +`--no-git-commit` skips automatic staging and committing, but still renders and +bumps non-autorelease specs. Upstream changes remain in the Git commit body, not the changelog comment. Commit refreshed TOML before rendering; configure Git author identity and install `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 6436007ab..37d8efb3c 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -86,8 +86,10 @@ calculation = "manual" ## Render configuration -Upstream rendering always runs host `rpmdev-bumpspec -c` before the existing -processing and automatically commits the component output directory. Commit +Upstream rendering runs host `rpmdev-bumpspec -c` only for non-autorelease specs +and automatically commits the component output directory. Missing +`--changelog-comment` opens `$EDITOR` or a local editor for non-autorelease, +or Git's own editor for an autorelease commit. Commit input TOML first, install host `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed, and configure Git author identity. Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded and the diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 05223da02..a7bc2e1ed 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -89,8 +89,8 @@ For lockless upstream components, prepare the changelog, then Release, using the upstream spec and last committed render before writing macros, applying overlays, and updating sources. Overlays can override the prepared release and changelog. Compare output; unchanged renders require --no-change-rebuild to record a timestamp. -Then run rpmdev-bumpspec -c with --changelog-comment (default: "update "). -Commit that directory with the same subject and upstream changes as the body. +Non-autorelease specs use rpmdev-bumpspec -c with --changelog-comment, or edited text. +Autorelease skips bumpspec; without --changelog-comment, Git prompts for a message. The first commit also includes all pending changes outside the rendered output directory. Later commits are component-only. --no-git-commit skips staging and committing. Output must be inside the project repository; configure Git author identity and @@ -163,8 +163,8 @@ func addRenderOptions(cmd *cobra.Command, options *RenderOptions) { cmd.Flags().BoolVar(&options.NoChangeRebuild, "no-change-rebuild", false, "allow unchanged lockless upstream renders by recording 'date -Is' output in '.no-change-rebuild'") cmd.Flags().StringVar(&options.ChangelogComment, "changelog-comment", "", - "comment passed verbatim to rpmdev-bumpspec -c and used as the Git commit subject "+ - "for lockless upstream components (default: update )") + "comment passed verbatim to rpmdev-bumpspec -c and used as the Git commit message "+ + "for lockless upstream components (if omitted, prompt in an editor)") cmd.Flags().BoolVar(&options.NoGitCommit, "no-git-commit", false, "leave lockless upstream rendered output uncommitted without staging it") cmd.Flags().BoolVar(&options.AllowUncommitted, "allow-uncommitted", false, @@ -500,10 +500,8 @@ func parallelPrepare( default: result.Value.prepared.index = idx if update := result.Value.prepared.update; update != nil { - if options.changelogCommentExplicit || options.ChangelogComment != "" { - update.subject = options.ChangelogComment - } - + update.subject = options.ChangelogComment + update.subjectProvided = options.changelogCommentExplicit || options.ChangelogComment != "" update.noGitCommit = options.NoGitCommit } @@ -738,7 +736,7 @@ func parallelFinish( parmapResults := parmap.Map( workerEnv, - env.IOBoundConcurrency(), + renderFinishConcurrency(env, prepared, checkOnly), prepared, func(done, _ int) { progressEvent.SetProgress(int64(done), total) }, func(_ context.Context, prep *preparedComponent) *RenderResult { diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 139051e0f..b5c107dba 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -802,8 +802,9 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { env.TestFS = comparisonFS called := false dated := false - wantBump := !testCase.checkOnly && !testCase.compareFail && !testCase.wantNoChanges - wantMarker := wantBump && !testCase.drift + wantFinish := !testCase.checkOnly && !testCase.compareFail && !testCase.wantNoChanges + wantBump := wantFinish && !testCase.autorelease + wantMarker := wantFinish && !testCase.drift env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { assert.True(t, comparisonFS.compared, "comparison must precede the bump") @@ -868,11 +869,13 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { assert.Equal(t, testCase.wantChanged, result.Changed) want := strings.Replace(original, "Release: 9\n", "Release: 10\n", 1) - if !wantBump { + if !wantFinish { want = actual staged, readErr := fileutils.ReadFile(baseFS, stagedSpec) require.NoError(t, readErr) assert.Equal(t, original, string(staged)) + } else if !wantBump { + want = original } output, readErr := fileutils.ReadFile(baseFS, outputSpec) @@ -881,7 +884,7 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { markerExists, readErr := fileutils.Exists(baseFS, markerPath) require.NoError(t, readErr) - assert.Equal(t, !wantBump || wantMarker, markerExists) + assert.Equal(t, !wantFinish || wantMarker, markerExists) if markerExists { marker, readErr := fileutils.ReadFile(baseFS, markerPath) diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 8ce1eebb7..5c77171df 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -36,11 +36,12 @@ const noChangeRebuildMarker = ".no-change-rebuild" var errNoRenderChanges = errors.New("no change since the last render; use '--no-change-rebuild' to rebuild anyway") type locklessRenderUpdate struct { - repoRoot string - path string - subject string - changes string - noGitCommit bool + repoRoot string + path string + subject string + changes string + noGitCommit bool + subjectProvided bool } type lastRender struct { @@ -118,7 +119,6 @@ func prepareLocklessRender( update := &locklessRenderUpdate{ repoRoot: root, path: path, - subject: "update " + comp.GetName(), } _, opened, err := openLocklessSpec(env, dir, comp.GetName(), editor) @@ -823,15 +823,36 @@ func writeNoChangeRebuild(env *azldev.Env, dir string) error { } func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate) (bool, error) { - if err := prereqs.RequireExecutable(env, sources.RPMDevBumpspecBinary, nil); err != nil { - return false, fmt.Errorf("lockless upstream rendering requires 'rpmdev-bumpspec'; install 'rpmdevtools':\n%w", err) + before, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return false, fmt.Errorf("reading spec before release bump:\n%w", err) } - before, err := fileutils.ReadFile(env.FS(), path) + opened, err := spec.OpenSpec(bytes.NewReader(before)) if err != nil { return false, fmt.Errorf("reading spec before release bump:\n%w", err) } + release, err := opened.GetLastTag("", "Release") + if err != nil { + return false, fmt.Errorf("reading Release before release bump:\n%w", err) + } + + if sources.ReleaseUsesAutorelease(release) { + return false, nil + } + + if err := prereqs.RequireExecutable(env, sources.RPMDevBumpspecBinary, nil); err != nil { + return false, fmt.Errorf("lockless upstream rendering requires 'rpmdev-bumpspec'; install 'rpmdevtools':\n%w", err) + } + + if !update.subjectProvided && update.subject == "" && !env.DryRun() { + update.subject, err = editRenderChangelog(env, path) + if err != nil { + return false, err + } + } + raw := exec.CommandContext(env, sources.RPMDevBumpspecBinary, "-c", update.subject, path) raw.Dir = filepath.Dir(path) @@ -953,15 +974,7 @@ func commitLocklessRender(env *azldev.Env, update *locklessRenderUpdate, include return false, nil } - args := []string{"commit", "--only", "-m", update.subject} - if update.changes != "" { - args = append(args, "-m", update.changes) - } - - args = append(args, "--") - - args = append(args, paths...) - if _, err := git.RunInDir(env, env, update.repoRoot, args...); err != nil { + if err := runLocklessRenderCommit(env, update, paths); err != nil { return false, fmt.Errorf("committing rendered component:\n%w", err) } diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index fc50bd814..f1bd269f7 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -508,15 +508,6 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { called = true assert.Equal(t, []string{"rpmdev-bumpspec", "-c", "update curl", path}, cmd.Args) - prepared, err := fileutils.ReadFile(env.TestFS, path) - require.NoError(t, err) - assert.Contains(t, string(prepared), "Release: "+testCase.wantRelease+"\n") - - if testCase.history.spec != "" && testCase.release != "%autorelease" { - assert.True(t, strings.HasSuffix(string(prepared), oldChangelog)) - } else { - assert.True(t, strings.HasSuffix(string(prepared), "* Upstream\n- New entry\n")) - } return nil } @@ -529,9 +520,19 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { require.NoError(t, err) assert.False(t, called, "carryover must not perform the final bump") + prepared, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Contains(t, string(prepared), "Release: "+testCase.wantRelease+"\n") + + if testCase.history.spec != "" && testCase.release != "%autorelease" { + assert.True(t, strings.HasSuffix(string(prepared), oldChangelog)) + } else { + assert.True(t, strings.HasSuffix(string(prepared), "* Upstream\n- New entry\n")) + } + _, err = bumpLocklessSpec(env.Env, path, update) require.NoError(t, err) - assert.True(t, called) + assert.Equal(t, !sources.ReleaseUsesAutorelease(testCase.wantRelease), called) assert.Equal(t, testCase.wantChanges, update.changes) if testCase.bootstrap { @@ -659,13 +660,6 @@ func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { bumped := false env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { bumped = true - content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") - require.NoError(t, err) - assert.Contains(t, string(content), "Release: "+testCase.wantRelease+"\n") - assert.Contains(t, string(content), "%changelog\n%autochangelog\n") - content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") - require.NoError(t, err) - assert.Equal(t, carriedAutoreleaseChangelog, string(content)) return nil } @@ -676,9 +670,17 @@ func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { require.NoError(t, err) assert.False(t, bumped) + content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: "+testCase.wantRelease+"\n") + assert.Contains(t, string(content), "%changelog\n%autochangelog\n") + content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, carriedAutoreleaseChangelog, string(content)) + _, err = bumpLocklessSpec(env.Env, "/working/curl.spec", update) require.NoError(t, err) - assert.True(t, bumped) + assert.False(t, bumped) assert.Equal(t, []string{"expand"}, stub.calls) assert.NotEqual(t, "/working", snapshot) @@ -1019,7 +1021,7 @@ func TestPrepareComponentSources_ChangelogAndReleaseBeforeOverlays(t *testing.T) upstreamVersion, overlayVersion, wantRelease string explicit, noGitCommit, skipReleaseOverlay bool }{ - {name: "default comment", wantComment: "update curl"}, + {name: "edited comment", wantComment: "Edited changelog text"}, { name: "custom comment", comment: "Fix curl's patches; keep $COMPONENT literal", wantComment: "Fix curl's patches; keep $COMPONENT literal", explicit: true, @@ -1030,11 +1032,11 @@ func TestPrepareComponentSources_ChangelogAndReleaseBeforeOverlays(t *testing.T) wantComment: "Prepare curl for review", noGitCommit: true, }, { - name: "version overlay cannot enable carryover", wantComment: "update curl", + name: "version overlay cannot enable carryover", comment: "update curl", wantComment: "update curl", upstreamVersion: "2", overlayVersion: "1", wantRelease: "1", skipReleaseOverlay: true, }, { - name: "version overlay cannot disable carryover", wantComment: "update curl", + name: "version overlay cannot disable carryover", comment: "update curl", wantComment: "update curl", upstreamVersion: "1", overlayVersion: "2", wantRelease: "9", skipReleaseOverlay: true, }, } { @@ -1133,7 +1135,14 @@ upstream-commit = "aaaaaaa" return hashes[0], nil } + env.CmdFactory.RegisterCommandInSearchPath("test-editor") + setRenderEditor(t, env, "test-editor") env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + if cmd.Args[0] == "test-editor" { + return fileutils.WriteFile(env.TestFS, cmd.Args[len(cmd.Args)-1], + []byte(testCase.wantComment), fileperms.PrivateFile) + } + bumped = true assert.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) diff --git a/internal/app/azldev/cmds/component/render_message.go b/internal/app/azldev/cmds/component/render_message.go new file mode 100644 index 000000000..598857827 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_message.go @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/kballard/go-shellquote" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" +) + +func renderFinishConcurrency(env *azldev.Env, prepared []*preparedComponent, checkOnly bool) int { + if !checkOnly && !env.DryRun() { + for _, prep := range prepared { + if prep.update != nil && !prep.update.subjectProvided && prep.update.subject == "" { + return 1 + } + } + } + + return env.IOBoundConcurrency() +} + +func renderEditor(env *azldev.Env) ([]string, error) { + if editor := env.OSEnv().Getenv("EDITOR"); editor != "" { + args, err := shellquote.Split(editor) + if err != nil { + return nil, fmt.Errorf("parsing 'EDITOR':\n%w", err) + } + + if len(args) == 0 || !env.CommandInSearchPath(args[0]) { + return nil, fmt.Errorf("editor configured by 'EDITOR' is unavailable: %#q", editor) + } + + return args, nil + } + + for _, editor := range []string{"editor", "sensible-editor", "vi", "nano"} { + if env.CommandInSearchPath(editor) { + return []string{editor}, nil + } + } + + return nil, errors.New("no local editor found; set 'EDITOR' or provide '--changelog-comment'") +} + +func editRenderChangelog(env *azldev.Env, specPath string) (string, error) { + args, err := renderEditor(env) + if err != nil { + return "", err + } + + path, cleanup, err := renderMessageFile(env, "") + if err != nil { + return "", err + } + defer cleanup() + + slog.Info("Enter the changelog text, then save and close the editor", "spec", specPath) + + raw := exec.CommandContext(env, args[0], append(args[1:], path)...) + if err := runRenderInteractive(env, raw); err != nil { + return "", fmt.Errorf("editing render changelog:\n%w", err) + } + + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return "", fmt.Errorf("reading edited render changelog:\n%w", err) + } + + if strings.TrimSpace(string(content)) == "" { + return "", errors.New("no changelog text provided; rendering will not be done") + } + + return string(content), nil +} + +func renderMessageFile(env *azldev.Env, content string) (path string, cleanup func(), err error) { + dir, err := fileutils.MkdirTempInTempDir(env.FS(), "azldev-render-message-") + if err != nil { + return "", nil, fmt.Errorf("creating render message directory:\n%w", err) + } + + cleanup = func() { + if err := env.FS().RemoveAll(dir); err != nil { + slog.Debug("Failed to remove render message directory", "path", dir, "error", err) + } + } + + path = filepath.Join(dir, "message.txt") + if err := fileutils.WriteFile(env.FS(), path, []byte(content), fileperms.PrivateFile); err != nil { + cleanup() + + return "", nil, fmt.Errorf("creating render message file:\n%w", err) + } + + return path, cleanup, nil +} + +func runRenderInteractive(env *azldev.Env, raw *exec.Cmd) error { + raw.Stdin = os.Stdin + raw.Stdout = os.Stdout + raw.Stderr = os.Stderr + + command, err := env.Command(raw) + if err != nil { + return fmt.Errorf("creating interactive render command:\n%w", err) + } + + if err := command.Run(env); err != nil { + return fmt.Errorf("running interactive render command:\n%w", err) + } + + return nil +} + +func runLocklessRenderCommit(env *azldev.Env, update *locklessRenderUpdate, paths []string) error { + args := []string{"commit", "--only"} + if update.subjectProvided || update.subject != "" { + args = append(args, "-m", update.subject) + if update.changes != "" { + args = append(args, "-m", update.changes) + } + + args = append(args, "--") + args = append(args, paths...) + + _, err := git.RunInDir(env, env, update.repoRoot, args...) + if err != nil { + return fmt.Errorf("running render commit:\n%w", err) + } + + return nil + } + + path, cleanup, err := renderMessageFile(env, "\n\n"+update.changes+"\n") + if err != nil { + return err + } + defer cleanup() + + args = append([]string{"-C", update.repoRoot}, args...) + args = append(args, "--template", path, "--") + args = append(args, paths...) + + return runRenderInteractive(env, exec.CommandContext(env, "git", args...)) +} diff --git a/internal/app/azldev/cmds/component/render_message_internal_test.go b/internal/app/azldev/cmds/component/render_message_internal_test.go new file mode 100644 index 000000000..7d759ba60 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_message_internal_test.go @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "os" + "os/exec" + "slices" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setRenderEditor(t *testing.T, env *testutils.TestEnv, editor string) { + t.Helper() + + osEnv, ok := env.TestOSEnv.(*testctx.TestOSEnv) + require.True(t, ok) + osEnv.SetEnv("EDITOR", editor) +} + +func TestRenderEditorSelection(t *testing.T) { + for _, testCase := range []struct { + name, editor, available, wantError string + want []string + }{ + { + name: "quoted arguments", editor: "edit-tool --wait 'two words'", available: "edit-tool", + want: []string{"edit-tool", "--wait", "two words"}, + }, + {name: "fallback", available: "vi", want: []string{"vi"}}, + {name: "invalid quoting", editor: "edit-tool '", wantError: "parsing 'EDITOR'"}, + {name: "unavailable configured editor", editor: "edit-tool", wantError: "unavailable"}, + {name: "no editor", wantError: "no local editor"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setRenderEditor(t, env, testCase.editor) + + if testCase.available != "" { + env.CmdFactory.RegisterCommandInSearchPath(testCase.available) + } + + args, err := renderEditor(env.Env) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + } else { + require.NoError(t, err) + assert.Equal(t, testCase.want, args) + } + }) + } +} + +func TestBumpLocklessSpecMessageAndAutorelease(t *testing.T) { + for _, testCase := range []struct { + name, release, message string + provided, autorelease bool + }{ + {name: "edited text", release: "1", message: "Fix feature\n\nKeep details.\n"}, + {name: "provided text", release: "1", message: "Explicit entry", provided: true}, + {name: "explicit empty value", release: "1", provided: true}, + {name: "autorelease without message", release: "%autorelease -b 9", autorelease: true}, + {name: "autorelease with message", release: "%autorelease", message: "Git only", provided: true, autorelease: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setRenderEditor(t, env, "edit-tool --wait") + env.CmdFactory.RegisterCommandInSearchPath("edit-tool") + + if !testCase.autorelease { + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + } + + const specPath = "/working/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, specPath, + []byte(renderTestSpec("1", testCase.release, "")), fileperms.PublicFile)) + + editCalls, bumpCalls := 0, 0 + + var messagePath string + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + switch cmd.Args[0] { + case "edit-tool": + editCalls++ + + assert.Equal(t, "--wait", cmd.Args[1]) + messagePath = cmd.Args[2] + assert.Same(t, os.Stdin, cmd.Stdin) + + return fileutils.WriteFile(env.TestFS, messagePath, []byte(testCase.message), fileperms.PrivateFile) + case "rpmdev-bumpspec": + bumpCalls++ + + assert.Equal(t, []string{"rpmdev-bumpspec", "-c", testCase.message, specPath}, cmd.Args) + default: + t.Fatalf("unexpected command: %v", cmd.Args) + } + + return nil + } + + update := &locklessRenderUpdate{subjectProvided: testCase.provided} + if testCase.provided { + update.subject = testCase.message + } + + _, err := bumpLocklessSpec(env.Env, specPath, update) + require.NoError(t, err) + + if testCase.autorelease { + assert.Zero(t, editCalls) + assert.Zero(t, bumpCalls) + } else { + assert.Equal(t, 1, bumpCalls) + assert.Equal(t, testCase.message, update.subject) + + if testCase.provided { + assert.Zero(t, editCalls) + } else { + assert.Equal(t, 1, editCalls) + + exists, err := fileutils.Exists(env.TestFS, messagePath) + require.NoError(t, err) + assert.False(t, exists) + } + } + }) + } +} + +func TestEditRenderChangelogRejectsEmptyAndFailedEditor(t *testing.T) { + editorErr := errors.New("editor failed") + + for _, fail := range []bool{false, true} { + t.Run(map[bool]string{false: "empty", true: "failure"}[fail], func(t *testing.T) { + env := testutils.NewTestEnv(t) + setRenderEditor(t, env, "edit-tool") + env.CmdFactory.RegisterCommandInSearchPath("edit-tool") + + var messagePath string + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + messagePath = cmd.Args[1] + + if fail { + return editorErr + } + + return nil + } + + _, err := editRenderChangelog(env.Env, "/working/curl.spec") + if fail { + require.ErrorIs(t, err, editorErr) + } else { + require.ErrorContains(t, err, "no changelog text") + } + + exists, err := fileutils.Exists(env.TestFS, messagePath) + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + +func TestLocklessRenderCommitPromptsWithoutMessage(t *testing.T) { + env := testutils.NewTestEnv(t) + + var templatePath string + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + assert.Equal(t, []string{"git", "-C", "/project", "commit", "--only"}, cmd.Args[:5]) + assert.NotContains(t, cmd.Args, "-m") + assert.Contains(t, cmd.Args, "SPECS/c/curl") + assert.Contains(t, cmd.Args, ":(top,literal)inputs.toml") + assert.Same(t, os.Stdin, cmd.Stdin) + assert.Same(t, os.Stdout, cmd.Stdout) + assert.Same(t, os.Stderr, cmd.Stderr) + templatePath = cmd.Args[slices.Index(cmd.Args, "--template")+1] + content, err := fileutils.ReadFile(env.TestFS, templatePath) + require.NoError(t, err) + assert.Contains(t, string(content), "abc123 Upstream fix") + + return nil + } + require.NoError(t, runLocklessRenderCommit(env.Env, + &locklessRenderUpdate{repoRoot: "/project", changes: "abc123 Upstream fix"}, + []string{"SPECS/c/curl", ":(top,literal)inputs.toml"})) + require.NotEmpty(t, templatePath) + exists, err := fileutils.Exists(env.TestFS, templatePath) + require.NoError(t, err) + assert.False(t, exists) +} diff --git a/internal/app/azldev/cmds/component/render_output_internal_test.go b/internal/app/azldev/cmds/component/render_output_internal_test.go index c5c774d9c..ce6e47279 100644 --- a/internal/app/azldev/cmds/component/render_output_internal_test.go +++ b/internal/app/azldev/cmds/component/render_output_internal_test.go @@ -50,7 +50,7 @@ func setupRenderOutput(t *testing.T, fileSystem afero.Fs, existing bool) { t.Helper() for name, content := range map[string]string{ - "curl.spec": "new spec", "sidecar": "new sidecar", ".git/HEAD": "upstream", + "curl.spec": "Name: curl\nVersion: 1\nRelease: 1\n", "sidecar": "new sidecar", ".git/HEAD": "upstream", } { require.NoError(t, fileutils.WriteFile(fileSystem, filepath.Join("/staging/curl", name), []byte(content), fileperms.PublicFile)) @@ -209,7 +209,7 @@ func TestCommitLocklessRenders_RestoresFailedOutput(t *testing.T) { content, err := fileutils.ReadFile(env.TestFS, filepath.Join(testRenderOutputDir, "curl.spec")) require.NoError(t, err) - assert.Equal(t, "new spec", string(content)) + assert.Equal(t, "Name: curl\nVersion: 1\nRelease: 1\n", string(content)) for _, name := range []string{".git", "extras", renderErrorMarkerFile} { exists, err := fileutils.Exists(env.TestFS, filepath.Join(testRenderOutputDir, name)) From 0a682b3b4a583857f042308c9032cf32b2fa2267 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 6 Nov 2026 00:01:10 +0000 Subject: [PATCH 34/70] fix(component): clean editor-provided changelog messages Seed the changelog editor with two blank lines and commented instructions. Discard blank and column-one comment lines from edited messages, and join the remaining lines without a trailing newline so rpmdev-bumpspec leaves only one blank line before the previous changelog entry. Reuse the cleaned text for Git, while preserving explicitly supplied changelog comments verbatim. Reject unchanged or comment-only templates. Update user and emitted agent guidance alongside the regression tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 15 +++-- .../app/azldev/agentskill/agentskill_test.go | 6 ++ .../azldev/agentskill/content/azldev.md.tmpl | 4 +- .../agentskill/content/comp-toml.md.tmpl | 3 + .../content/refresh-upstream-commit.md.tmpl | 8 ++- .../content/withoutlockfile/azldev.md.tmpl | 4 +- .../content/withoutlockfile/comp-toml.md.tmpl | 7 +- .../azldev/cmds/component/render_message.go | 27 ++++++-- .../component/render_message_internal_test.go | 65 +++++++++++++++++-- 9 files changed, 117 insertions(+), 22 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 331298408..c01499017 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -269,11 +269,16 @@ Changed renders, or explicitly requested no-change rebuilds, then invoke host `rpmdev-bumpspec -c` **only for non-autorelease specs**, with the exact `--changelog-comment` value. There is no default comment. If the flag is omitted, rendering opens `$EDITOR` (including configured arguments), or a local editor -(`editor`, `sensible-editor`, `vi`, or `nano`), to obtain the entry. Save and close -the editor to continue. Its text is passed unchanged to `rpmdev-bumpspec -c` and -reused as the Git commit message. Empty edited text or editor failure aborts -rendering before output replacement. Missing-comment finishing is serialized -to prevent simultaneous editors. +(`editor`, `sensible-editor`, `vi`, or `nano`), to obtain the entry. The editor +starts with two blank lines followed by `#`-commented instructions. Save and close +the editor to continue. Blank and whitespace-only lines and lines starting with +`#` are removed; other message lines are preserved, with no trailing newline. +The cleaned text is passed to `rpmdev-bumpspec -c` and reused as the Git commit +message, leaving a single blank line between the new and previous changelog +entries. Explicit `--changelog-comment` values remain unchanged. +An edited message containing only ignored lines or editor failure aborts rendering +before output replacement. Missing-comment finishing is serialized to prevent +simultaneous editors. Autorelease specs skip `rpmdev-bumpspec` entirely. With `--changelog-comment`, Git uses the supplied message; without it, Git opens its own configured editor diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index a8dada91e..e9d9c28f4 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -167,6 +167,8 @@ func TestComponentTomlSkillDocumentsPerComponentRenderModes(t *testing.T) { assert.Contains(t, doc, "`lockless-components.txt` beside the top-level `azldev.toml`") assert.Contains(t, doc, "all others use lock files") assert.Contains(t, doc, "for every command") + assert.Contains(t, doc, "two blank lines and `#`-commented instructions") + assert.Contains(t, doc, "Blank lines and lines starting with `#` are removed, with no trailing newline") } } @@ -195,6 +197,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "`--changelog-comment`") assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") assert.Contains(t, doc, "`$EDITOR` or a local editor") + assert.Contains(t, doc, "two blank lines and `#`-commented instructions") + assert.Contains(t, doc, "Blank lines and lines starting with `#` are removed, with no trailing newline") assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") assert.Contains(t, doc, "`azldev comp make-lockless`") assert.Contains(t, doc, "spaces and tabs after `Release:`") @@ -238,6 +242,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "`--changelog-comment`") assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") assert.Contains(t, doc, "`$EDITOR` or a local editor") + assert.Contains(t, doc, "two blank lines and `#`-commented instructions") + assert.Contains(t, doc, "Blank lines and lines starting with `#` are removed, with no trailing newline") assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") assert.Contains(t, doc, "`azldev comp make-lockless`") assert.Contains(t, doc, "spaces and tabs after `Release:`") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 17536c138..a3404e234 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -74,7 +74,9 @@ the rendered output directory. Other rendered paths are excluded even if staged; later commits include only their component's dist-git changes. `--changelog-comment` supplies the exact `rpmdev-bumpspec -c` value and commit message, with no default. If omitted, `$EDITOR` or a local editor -collects the entry; the exact edited text is reused for the Git message. +collects the entry using a template with two blank lines and `#`-commented instructions. +Blank lines and lines starting with `#` are removed, with no trailing newline; +the cleaned text is reused for the Git message. Explicit flag values remain unchanged. Autorelease skips `rpmdev-bumpspec`; without a supplied comment, Git prompts in its own editor for a message, with upstream changes in the template. Provide `--changelog-comment` for unattended rendering. diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index e51203819..c7ccc42d4 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -96,6 +96,9 @@ used outside the lockless upstream render workflow. That workflow always runs host `rpmdev-bumpspec -c` only for non-autorelease specs and automatically commits the component output. Missing `--changelog-comment` opens `$EDITOR` or a local editor for non-autorelease, or Git's own editor for an autorelease commit. +The changelog editor starts with two blank lines and `#`-commented instructions. +Blank lines and lines starting with `#` are removed, with no trailing newline; +the cleaned text is reused for the Git message. Explicit flag values remain unchanged. Lockless rendering skips synthetic history and its legacy release adjustment. Commit input TOML first. The first automatic commit also includes pending repository changes outside the diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 42f4b8f9d..584fca31a 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -139,8 +139,9 @@ other macro options. This includes legacy expansion. Unchanged-version modern ou still carries an explicitly committed `-b 1` as-is. The final host -`rpmdev-bumpspec -c` uses the exact `--changelog-comment` value, defaulting to -`update `. Upstream changes are not appended to this comment. +`rpmdev-bumpspec -c` uses the exact `--changelog-comment` value or the cleaned +editor-provided entry. There is no default comment. +Upstream changes are not appended to this comment. All prepared dist-git files except `.git` are retained, without file filtering or `spectool` queries. Synthetic history and its legacy release adjustment are skipped during lockless rendering. @@ -160,6 +161,9 @@ rendered output directory. Other rendered paths are excluded even if staged; later commits include only their component's dist-git changes. The message is the supplied changelog comment or, for non-autorelease specs, the entry collected through `$EDITOR` or a local editor. There is no default. +The changelog editor starts with two blank lines and `#`-commented instructions. +Blank lines and lines starting with `#` are removed, with no trailing newline; +the cleaned text is reused for the Git message. Explicit flag values remain unchanged. Autorelease skips `rpmdev-bumpspec`; without a supplied comment, Git prompts in its own editor for a message. The body/template contains upstream changes. Provide `--changelog-comment` for unattended rendering. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 584455236..0ae80cdf8 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -68,7 +68,9 @@ Other rendered paths are excluded even if staged; later commits include only their component's dist-git changes. `--changelog-comment` supplies the exact `rpmdev-bumpspec -c` value and commit message, with no default. If omitted, `$EDITOR` or a local editor collects the -entry; the exact edited text is reused for the Git message. +entry using a template with two blank lines and `#`-commented instructions. +Blank lines and lines starting with `#` are removed, with no trailing newline; +the cleaned text is reused for the Git message. Explicit flag values remain unchanged. Autorelease skips `rpmdev-bumpspec`; without a supplied comment, Git prompts in its own editor for a message, with upstream changes in the template. Provide `--changelog-comment` for unattended rendering. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 37d8efb3c..4b4c01dca 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -89,8 +89,11 @@ calculation = "manual" Upstream rendering runs host `rpmdev-bumpspec -c` only for non-autorelease specs and automatically commits the component output directory. Missing `--changelog-comment` opens `$EDITOR` or a local editor for non-autorelease, -or Git's own editor for an autorelease commit. Commit -input TOML first, install host `rpmdevtools`, `rpm-build`, and `rpmautospec` as +or Git's own editor for an autorelease commit. +The changelog editor starts with two blank lines and `#`-commented instructions. +Blank lines and lines starting with `#` are removed, with no trailing newline; +the cleaned text is reused for the Git message. Explicit flag values remain unchanged. +Commit input TOML first, install host `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed, and configure Git author identity. Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded and the `changelog` sidecar is preserved. Synthetic history and its legacy release diff --git a/internal/app/azldev/cmds/component/render_message.go b/internal/app/azldev/cmds/component/render_message.go index 598857827..428d62c34 100644 --- a/internal/app/azldev/cmds/component/render_message.go +++ b/internal/app/azldev/cmds/component/render_message.go @@ -19,6 +19,10 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" ) +const renderChangelogTemplate = "\n\n" + + "# Please enter a changelog entry message (one or more lines).\n" + + "# Blank lines and lines starting with # will be ignored.\n" + func renderFinishConcurrency(env *azldev.Env, prepared []*preparedComponent, checkOnly bool) int { if !checkOnly && !env.DryRun() { for _, prep := range prepared { @@ -60,13 +64,14 @@ func editRenderChangelog(env *azldev.Env, specPath string) (string, error) { return "", err } - path, cleanup, err := renderMessageFile(env, "") + path, cleanup, err := renderMessageFile(env, renderChangelogTemplate) if err != nil { return "", err } defer cleanup() - slog.Info("Enter the changelog text, then save and close the editor", "spec", specPath) + slog.Info("Enter changelog text, save and close the editor; blank lines and lines starting with # are ignored", + "spec", specPath) raw := exec.CommandContext(env, args[0], append(args[1:], path)...) if err := runRenderInteractive(env, raw); err != nil { @@ -78,11 +83,25 @@ func editRenderChangelog(env *azldev.Env, specPath string) (string, error) { return "", fmt.Errorf("reading edited render changelog:\n%w", err) } - if strings.TrimSpace(string(content)) == "" { + message := cleanRenderChangelog(string(content)) + if message == "" { return "", errors.New("no changelog text provided; rendering will not be done") } - return string(content), nil + return message, nil +} + +func cleanRenderChangelog(content string) string { + var lines []string + + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSuffix(line, "\r") + if strings.TrimSpace(line) != "" && !strings.HasPrefix(line, "#") { + lines = append(lines, line) + } + } + + return strings.Join(lines, "\n") } func renderMessageFile(env *azldev.Env, content string) (path string, cleanup func(), err error) { diff --git a/internal/app/azldev/cmds/component/render_message_internal_test.go b/internal/app/azldev/cmds/component/render_message_internal_test.go index 7d759ba60..7ac2a0532 100644 --- a/internal/app/azldev/cmds/component/render_message_internal_test.go +++ b/internal/app/azldev/cmds/component/render_message_internal_test.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "slices" + "strings" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" @@ -61,11 +62,15 @@ func TestRenderEditorSelection(t *testing.T) { func TestBumpLocklessSpecMessageAndAutorelease(t *testing.T) { for _, testCase := range []struct { - name, release, message string - provided, autorelease bool + name, release, message, wantMessage string + provided, autorelease bool }{ - {name: "edited text", release: "1", message: "Fix feature\n\nKeep details.\n"}, + { + name: "edited text", release: "1", message: "Fix feature\n\nKeep details.\n", + wantMessage: "Fix feature\nKeep details.", + }, {name: "provided text", release: "1", message: "Explicit entry", provided: true}, + {name: "provided multiline text", release: "1", message: "# Explicit\n\nKeep details.\n", provided: true}, {name: "explicit empty value", release: "1", provided: true}, {name: "autorelease without message", release: "%autorelease -b 9", autorelease: true}, {name: "autorelease with message", release: "%autorelease", message: "Git only", provided: true, autorelease: true}, @@ -85,6 +90,11 @@ func TestBumpLocklessSpecMessageAndAutorelease(t *testing.T) { editCalls, bumpCalls := 0, 0 + wantMessage := testCase.message + if !testCase.provided && !testCase.autorelease { + wantMessage = testCase.wantMessage + } + var messagePath string env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { @@ -96,11 +106,18 @@ func TestBumpLocklessSpecMessageAndAutorelease(t *testing.T) { messagePath = cmd.Args[2] assert.Same(t, os.Stdin, cmd.Stdin) - return fileutils.WriteFile(env.TestFS, messagePath, []byte(testCase.message), fileperms.PrivateFile) + content, err := fileutils.ReadFile(env.TestFS, messagePath) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(string(content), "\n\n#")) + assert.Contains(t, string(content), "enter a changelog entry message") + assert.Contains(t, string(content), "Blank lines and lines starting with # will be ignored") + + return fileutils.WriteFile(env.TestFS, messagePath, + []byte(testCase.message+string(content)), fileperms.PrivateFile) case "rpmdev-bumpspec": bumpCalls++ - assert.Equal(t, []string{"rpmdev-bumpspec", "-c", testCase.message, specPath}, cmd.Args) + assert.Equal(t, []string{"rpmdev-bumpspec", "-c", wantMessage, specPath}, cmd.Args) default: t.Fatalf("unexpected command: %v", cmd.Args) } @@ -121,7 +138,7 @@ func TestBumpLocklessSpecMessageAndAutorelease(t *testing.T) { assert.Zero(t, bumpCalls) } else { assert.Equal(t, 1, bumpCalls) - assert.Equal(t, testCase.message, update.subject) + assert.Equal(t, wantMessage, update.subject) if testCase.provided { assert.Zero(t, editCalls) @@ -137,11 +154,45 @@ func TestBumpLocklessSpecMessageAndAutorelease(t *testing.T) { } } +func TestCleanRenderChangelog(t *testing.T) { + for _, testCase := range []struct { + name string + lines []string + want string + crlf bool + }{ + { + name: "comments and blank lines", + lines: []string{"", "", "# Instructions", "Fix feature", " \t ", "Keep details.", "# Comment", ""}, + want: "Fix feature\nKeep details.", + }, + { + name: "preserve message whitespace and hashes", + lines: []string{" # Indented message", "Keep # inline hash "}, + want: " # Indented message\nKeep # inline hash ", + }, + {name: "only ignored lines", lines: []string{"", "\t", "# Instructions", "#", ""}}, + { + name: "CRLF", lines: []string{"", "# Instructions", "Fix feature", "", "Keep details.", ""}, + want: "Fix feature\nKeep details.", crlf: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + content := strings.Join(testCase.lines, "\n") + if testCase.crlf { + content = strings.ReplaceAll(content, "\n", "\r\n") + } + + assert.Equal(t, testCase.want, cleanRenderChangelog(content)) + }) + } +} + func TestEditRenderChangelogRejectsEmptyAndFailedEditor(t *testing.T) { editorErr := errors.New("editor failed") for _, fail := range []bool{false, true} { - t.Run(map[bool]string{false: "empty", true: "failure"}[fail], func(t *testing.T) { + t.Run(map[bool]string{false: "unchanged template", true: "failure"}[fail], func(t *testing.T) { env := testutils.NewTestEnv(t) setRenderEditor(t, env, "edit-tool") env.CmdFactory.RegisterCommandInSearchPath("edit-tool") From 94dfa2f5f2bbb0c893604bb891465f1b45239766 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 6 Nov 2026 01:39:12 +0000 Subject: [PATCH 35/70] fix(component): recover missing upstream render history Resolve the previous render's upstream commit before collecting its log. Fetch the exact commit from origin when the single-branch checkout lacks it, and retry resolution even if fetching fails. If the commit remains unavailable, record the component name and current upstream hash instead of aborting the render. Let failed historical branch fetches use the same recovery path, while preserving cancellation and unrelated Git log errors. Cover recovery and fallback with unit tests and update user and emitted agent documentation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 5 + .../app/azldev/agentskill/agentskill_test.go | 2 + .../content/refresh-upstream-commit.md.tmpl | 5 + .../azldev/cmds/component/render_lockless.go | 38 ++++- .../render_lockless_internal_test.go | 158 +++++++++++++++++- 5 files changed, 200 insertions(+), 8 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index c01499017..9170cceba 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -244,6 +244,11 @@ fetching the historical branch does not change the working checkout. Different pins, missing render history, or an error loading the old pin mean upstream changed. It records `git log --oneline ..` from the upstream checkout; without an old pin it records a one-line version update. +If the old pin cannot be resolved in the single-branch checkout, rendering +fetches that specific commit hash from `origin` and checks again. If the commit +remains unavailable, including after a failed historical-branch fetch, rendering +continues with `update to upstream commit ` as the +upstream changes text. Other errors collecting the Git log remain fatal. An error reading the current pin is fatal. It then compares the prepared tree with existing output, ignoring the root `.git` diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index e9d9c28f4..85d34438e 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -227,6 +227,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "retains `%autorelease`, `%autochangelog`") assert.Contains(t, doc, "effective upstream distro's `dist-git-branch`") assert.Contains(t, doc, "fetches that historical branch before comparing commits") + assert.Contains(t, doc, "fetches that specific commit hash") + assert.Contains(t, doc, "update to upstream commit ") assert.Contains(t, doc, "All prepared dist-git files except `.git` are retained") assert.Contains(t, doc, "immediately after checking out the pinned upstream commit") assert.Contains(t, doc, "that step never repeats the provenance query") diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 584fca31a..6564c2e3c 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -73,6 +73,11 @@ its full history rather than making a shallow clone. If the last render's committed TOML resolves to a different branch and the pins differ, rendering fetches that historical branch before comparing commits, without switching the working checkout. +If the old pin cannot be resolved, rendering fetches that specific commit hash +from `origin` and checks again. If it remains unavailable, rendering continues +with `update to upstream commit ` as the upstream +changes text, even when the historical branch fetch failed. Other Git log errors +remain fatal. With `build.emit-upstream-provenance`, rendering captures Fedora provenance immediately after checking out the pinned upstream commit, before diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 5c77171df..d6c7c721b 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -411,10 +411,23 @@ func collectRenderUpstreamChanges( if history.previousBranch != "" && history.previousBranch != history.currentBranch { refspec := "+refs/heads/" + history.previousBranch + ":refs/remotes/origin/" + history.previousBranch if _, err := git.RunInDir(env, env, dir, "fetch", "--no-tags", "--", "origin", refspec); err != nil { - return fmt.Errorf("fetching last render upstream branch %#q:\n%w", history.previousBranch, err) + slog.Warn("Cannot fetch last render upstream branch", + "component", name, "branch", history.previousBranch, "error", err) } } + if err := resolveRenderUpstreamCommit(env, dir, history.previous); err != nil { + if env.Err() != nil { + return fmt.Errorf("resolving last render upstream commit:\n%w", env.Err()) + } + + slog.Warn("Last render upstream commit is unavailable; using current upstream pin", + "component", name, "commit", history.previous, "error", err) + update.changes = fmt.Sprintf("update to upstream %s commit %s", name, history.current) + + return nil + } + changes, err := git.RunInDir(env, env, dir, "log", "--oneline", "--end-of-options", history.previous+".."+history.current, "--") if err != nil { @@ -426,6 +439,29 @@ func collectRenderUpstreamChanges( return nil } +func resolveRenderUpstreamCommit(env *azldev.Env, dir, commit string) error { + if _, err := git.RunInDir(env, env, dir, + "rev-parse", "--verify", "--end-of-options", commit+"^{commit}"); err == nil { + return nil + } + + if env.Err() != nil { + return fmt.Errorf("resolving upstream commit %#q:\n%w", commit, env.Err()) + } + + if _, err := git.RunInDir(env, env, dir, "fetch", "--no-tags", "--", "origin", commit); err != nil { + slog.Warn("Cannot fetch last render upstream commit", "commit", commit, "error", err) + } + + _, err := git.RunInDir(env, env, dir, + "rev-parse", "--verify", "--end-of-options", commit+"^{commit}") + if err != nil { + return fmt.Errorf("resolving upstream commit %#q after fetching:\n%w", commit, err) + } + + return nil +} + func locklessRenderRelease( env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, versionChanged bool, diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index f1bd269f7..d046396b7 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -33,6 +33,8 @@ import ( "go.uber.org/mock/gomock" ) +const renderTestRevParse = "rev-parse" + func renderTestSpec(version, release, changelog string) string { return fmt.Sprintf("Name: curl\nVersion: %s\nRelease: %s\nSummary: Test\nLicense: MIT\n"+ "\n%%description\nTest\n\n%%changelog\n%s", version, release, changelog) @@ -268,7 +270,7 @@ func TestCollectRenderUpstreamChanges_Branches(t *testing.T) { }, { name: "branch fetch fails", previous: "old", current: "new", - oldBranch: "f40", newBranch: "f41", wantFetch: true, fetchFails: true, + oldBranch: "f40", newBranch: "f41", wantFetch: true, fetchFails: true, wantLog: true, }, { name: "same commit on different branch", previous: "same", current: "same", @@ -283,6 +285,14 @@ func TestCollectRenderUpstreamChanges_Branches(t *testing.T) { var fetched, logged bool env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[3] == renderTestRevParse { + assert.Equal(t, []string{ + "git", "-C", "/working", renderTestRevParse, "--verify", "--end-of-options", "old^{commit}", + }, cmd.Args) + + return "old\n", nil + } + if cmd.Args[3] == "fetch" { fetched = true @@ -314,12 +324,7 @@ func TestCollectRenderUpstreamChanges_Branches(t *testing.T) { previous: testCase.previous, current: testCase.current, previousBranch: testCase.oldBranch, currentBranch: testCase.newBranch, }, update) - if testCase.fetchFails { - require.ErrorIs(t, err, fetchErr) - require.ErrorContains(t, err, "fetching last render upstream branch") - } else { - require.NoError(t, err) - } + require.NoError(t, err) assert.Equal(t, testCase.wantFetch, fetched) assert.Equal(t, testCase.wantLog, logged) @@ -331,6 +336,141 @@ func TestCollectRenderUpstreamChanges_Branches(t *testing.T) { } } +func TestCollectRenderUpstreamChanges_MissingCommit(t *testing.T) { + for _, testCase := range []struct { + name string + present, recovered, fetchFail bool + logFail, cancelResolve bool + branchFail bool + }{ + {name: "already available", present: true}, + {name: "fetched commit", recovered: true}, + {name: "failed fetch", fetchFail: true}, + {name: "failed fetch still recovers commit", recovered: true, fetchFail: true}, + {name: "successful fetch without commit"}, + {name: "failed branch and commit fetch", branchFail: true, fetchFail: true}, + {name: "commit recovered after branch fetch failure", branchFail: true, recovered: true}, + {name: "log failure is fatal", present: true, logFail: true}, + {name: "cancelled resolution", cancelResolve: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + workerEnv, cancel := env.Env.WithCancel() + defer cancel() + + const ( + previous = "1111111111111111111111111111111111111111" + current = "2222222222222222222222222222222222222222" + ) + + logErr := errors.New("log failed") + + var ( + commands []string + resolutions int + ) + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + commands = append(commands, cmd.Args[3]) + + switch cmd.Args[3] { + case renderTestRevParse: + resolutions++ + + assert.Equal(t, []string{ + "git", "-C", "/working", renderTestRevParse, "--verify", "--end-of-options", previous + "^{commit}", + }, cmd.Args) + + if testCase.cancelResolve { + cancel() + + return "", context.Canceled + } + + if testCase.present || (resolutions == 2 && testCase.recovered) { + return previous, nil + } + + return "", errors.New("unknown revision") + case "fetch": + if strings.HasPrefix(cmd.Args[7], "+refs/heads/") { + assert.Equal(t, "+refs/heads/f40:refs/remotes/origin/f40", cmd.Args[7]) + + return "", errors.New("branch unavailable") + } + + assert.Equal(t, []string{ + "git", "-C", "/working", "fetch", "--no-tags", "--", "origin", previous, + }, cmd.Args) + + if testCase.fetchFail { + return "", errors.New("fetch failed") + } + + return "", nil + case "log": + assert.Equal(t, []string{ + "git", "-C", "/working", "log", "--oneline", "--end-of-options", previous + ".." + current, "--", + }, cmd.Args) + + if testCase.logFail { + return "", logErr + } + + return "def456 Fetched upstream change\n", nil + default: + t.Fatalf("unexpected command: %v", cmd.Args) + + return "", nil + } + } + + update := &locklessRenderUpdate{} + + history := lastRender{previous: previous, current: current} + if testCase.branchFail { + history.previousBranch, history.currentBranch = "f40", "f41" + } + + err := collectRenderUpstreamChanges(workerEnv, "/working", "rust-asyncband", "1", + history, update) + + switch { + case testCase.cancelResolve: + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, []string{renderTestRevParse}, commands) + assert.Empty(t, update.changes) + case testCase.logFail: + require.ErrorIs(t, err, logErr) + require.ErrorContains(t, err, "collecting upstream changes") + assert.Equal(t, []string{renderTestRevParse, "log"}, commands) + default: + require.NoError(t, err) + + expected := []string{renderTestRevParse} + if testCase.branchFail { + expected = append([]string{"fetch"}, expected...) + } + + if !testCase.present { + expected = append(expected, "fetch", renderTestRevParse) + } + + if testCase.present || testCase.recovered { + expected = append(expected, "log") + + assert.Equal(t, "def456 Fetched upstream change", update.changes) + } else { + assert.Equal(t, "update to upstream rust-asyncband commit "+current, update.changes) + } + + assert.Equal(t, expected, commands) + } + }) + } +} + func TestReadLastRender_AbsentAndInvalidHistory(t *testing.T) { valid := []byte(`[project] default-distro = { name = "test", version = "1" } @@ -495,6 +635,10 @@ func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { fileperms.PublicFile)) env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[3] == renderTestRevParse { + return "old\n", nil + } + assert.Equal(t, []string{ "git", "-C", "/working", "log", "--oneline", "--end-of-options", "old..new", "--", From 5ea60c8711f22d0206dea625b8ae9bb3169a8c08 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 6 Nov 2026 03:44:38 +0000 Subject: [PATCH 36/70] fix(component): sort lockless component selections Merge selected component names into lockless-components.txt and write the complete list in lexical order, one exact name per line without blank lines or duplicates. Normalize existing entries even when the selected components are already listed, while leaving lock files untouched. Cover insertion, whitespace cleanup, deduplication, and repeated selection with regression tests. Update user and emitted agent guidance and regenerate the CLI reference. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 8 ++++--- .../cli/azldev_component_make-lockless.md | 5 +++-- .../app/azldev/agentskill/agentskill_test.go | 2 ++ .../azldev/agentskill/content/azldev.md.tmpl | 3 ++- .../content/refresh-upstream-commit.md.tmpl | 3 ++- .../content/withoutlockfile/azldev.md.tmpl | 3 ++- .../app/azldev/cmds/component/makelockless.go | 22 +++++++------------ .../cmds/component/makelockless_test.go | 12 ++++++++-- 8 files changed, 34 insertions(+), 24 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 9170cceba..f793ee7e8 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -45,9 +45,11 @@ azldev comp make-lockless curl rust-asyncband ``` `make-lockless` accepts the normal component selectors (`-p`, `-g`, and `-a`). -It appends missing exact names to the top-level list without rewriting existing -entries or removing lock files, then runs the `refresh-upstream-commit` pipeline -for the selection. Already-listed components are refreshed again without adding +It merges selected exact names into the top-level list and writes the complete +list in lexical order, one name per line without blank lines or duplicates. +Existing names are retained and lock files are left untouched. It then runs the +`refresh-upstream-commit` pipeline for the selection. +Already-listed components are refreshed again without adding duplicates. Non-upstream components are listed too; refresh removes any generated pin for them without contacting an upstream provider. If refreshing fails, the list stays updated and successful pins are saved. diff --git a/docs/user/reference/cli/azldev_component_make-lockless.md b/docs/user/reference/cli/azldev_component_make-lockless.md index 73c2df064..f9bafc00e 100644 --- a/docs/user/reference/cli/azldev_component_make-lockless.md +++ b/docs/user/reference/cli/azldev_component_make-lockless.md @@ -9,8 +9,9 @@ Select the lockless workflow and refresh upstream commits Add selected component names to lockless-components.txt beside the top-level azldev.toml, then run the refresh-upstream-commit pipeline for those components. -Existing entries, whitespace, and ordering are preserved. Missing names are -appended once, using exact component names rather than selection patterns. +Names are written in lexical order, one exact component name per line, +without blank lines or duplicates. Existing names are retained; selection +patterns are expanded to exact component names. Existing lock files are left untouched. Already-listed components are refreshed again without duplicating their entries. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 85d34438e..86289a919 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -201,6 +201,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "Blank lines and lines starting with `#` are removed, with no trailing newline") assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") assert.Contains(t, doc, "`azldev comp make-lockless`") + assert.Contains(t, doc, "lexical order") + assert.Contains(t, doc, "without blank lines or duplicates") assert.Contains(t, doc, "spaces and tabs after `Release:`") } diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index a3404e234..cbc72232e 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -38,7 +38,8 @@ images assemble RPMs into bootable artifacts. workflow per component for all commands. Listed components ignore locks and use generated upstream-commit TOML instead. All others use locks. `azldev comp make-lockless` adds selected exact names to that list if needed, -then runs the refresh pipeline. Existing list contents and lock files are retained. +then runs the refresh pipeline. The list is written in lexical order, one name +per line without blank lines or duplicates. Existing names and lock files are retained. Already-listed components are still refreshed; a refresh failure leaves the list updated. Before preparing any selected lockless upstream component, rendering requires a diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 6564c2e3c..153e47e9d 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -44,7 +44,8 @@ Select this workflow by listing the component's exact name in `lockless-components.txt` beside the top-level `azldev.toml`. Alternatively, `azldev comp make-lockless` adds selected exact names if needed and runs this refresh pipeline. It accepts `-p`, `-g`, `-a`, or positional names, -preserves existing list contents and lock files, and refreshes already-listed components. +writes the list in lexical order, one name per line without blank lines or duplicates, +preserves existing names and lock files, and refreshes already-listed components. A refresh failure leaves the updated list in place; fix the failure and rerun. Refresh skips unlisted components and preserves their generated TOML files. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 0ae80cdf8..20ae46a70 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -35,7 +35,8 @@ images assemble RPMs into bootable artifacts. `azldev comp make-lockless` adds selected exact names to the top-level `lockless-components.txt` if needed, then runs the refresh pipeline. -Existing list contents and lock files are retained. Already-listed components +The list is written in lexical order, one name per line without blank lines or duplicates. +Existing names and lock files are retained. Already-listed components are still refreshed; a refresh failure leaves the list updated. Before preparing any upstream component, rendering requires a clean project Git diff --git a/internal/app/azldev/cmds/component/makelockless.go b/internal/app/azldev/cmds/component/makelockless.go index c89b2dabb..44e29e563 100644 --- a/internal/app/azldev/cmds/component/makelockless.go +++ b/internal/app/azldev/cmds/component/makelockless.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log/slog" + "maps" "os" "path/filepath" "slices" @@ -33,8 +34,9 @@ func NewMakeLocklessCmd() *cobra.Command { Long: `Add selected component names to lockless-components.txt beside the top-level azldev.toml, then run the refresh-upstream-commit pipeline for those components. -Existing entries, whitespace, and ordering are preserved. Missing names are -appended once, using exact component names rather than selection patterns. +Names are written in lexical order, one exact component name per line, +without blank lines or duplicates. Existing names are retained; selection +patterns are expanded to exact component names. Existing lock files are left untouched. Already-listed components are refreshed again without duplicating their entries. @@ -118,22 +120,14 @@ func addLocklessComponents(env *azldev.Env, names []string) error { return fmt.Errorf("reading %#q:\n%w", path, err) } - var additions strings.Builder - for _, name := range names { - if !selected[name] { - additions.WriteString(name + "\n") - selected[name] = true - } + selected[name] = true } - if additions.Len() > 0 { - if len(content) > 0 && content[len(content)-1] != '\n' { - content = append(content, '\n') - } + updated := strings.Join(slices.Sorted(maps.Keys(selected)), "\n") + "\n" - content = append(content, additions.String()...) - if err := fileutils.WriteFile(env.FS(), path, content, fileperms.PublicFile); err != nil { + if string(content) != updated { + if err := fileutils.WriteFile(env.FS(), path, []byte(updated), fileperms.PublicFile); err != nil { return fmt.Errorf("writing %#q:\n%w", path, err) } } diff --git a/internal/app/azldev/cmds/component/makelockless_test.go b/internal/app/azldev/cmds/component/makelockless_test.go index 9d43cbeab..62c703e50 100644 --- a/internal/app/azldev/cmds/component/makelockless_test.go +++ b/internal/app/azldev/cmds/component/makelockless_test.go @@ -48,9 +48,17 @@ func TestMakeComponentsLockless_SelectsAndRefreshes(t *testing.T) { name, existing, want string }{ {name: "missing file", want: "curl\nrust-asyncband\n"}, - {name: "missing newline", existing: "rust-asyncband", want: "rust-asyncband\ncurl\n"}, - {name: "existing whitespace", existing: "\t rust-asyncband \r\n\n", want: "\t rust-asyncband \r\n\ncurl\n"}, + {name: "missing newline", existing: "rust-asyncband", want: "curl\nrust-asyncband\n"}, + {name: "existing whitespace", existing: "\t rust-asyncband \r\n\n", want: "curl\nrust-asyncband\n"}, + {name: "blank file", existing: "\n \t\r\n\n", want: "curl\nrust-asyncband\n"}, + {name: "insert before existing names", existing: "rust-asyncband\nvim\n", want: "curl\nrust-asyncband\nvim\n"}, + {name: "insert between existing names", existing: "curl\nvim\n", want: "curl\nrust-asyncband\nvim\n"}, {name: "already selected", existing: "curl\nrust-asyncband\n", want: "curl\nrust-asyncband\n"}, + {name: "already selected unsorted", existing: "rust-asyncband\ncurl\n", want: "curl\nrust-asyncband\n"}, + { + name: "already selected duplicates", existing: "rust-asyncband\ncurl\nrust-asyncband\ncurl\n", + want: "curl\nrust-asyncband\n", + }, } { t.Run(testCase.name, func(t *testing.T) { env := testutils.NewTestEnv(t) From ffc05e9050558e02356a52935f15eb0309ebf37a Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 6 Nov 2026 06:25:50 +0000 Subject: [PATCH 37/70] feat(component): extract standalone dist-git repositories Add comp dist-git-repo with a required component and output directory. Require git-filter-repo, clone the current branch with --single-branch and --no-local, and filter the configured rendered component subtree. Protect existing output unless --force is provided, validate replacement paths before deletion, and document the workflow in user and agent guides. Add unit coverage and verify behavior with native Git smoke tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/README.md | 1 + docs/user/how-to/extract-dist-git-repo.md | 65 +++++ docs/user/reference/cli/azldev_component.md | 1 + .../cli/azldev_component_dist-git-repo.md | 64 ++++ .../app/azldev/agentskill/agentskill_test.go | 11 + .../azldev/agentskill/content/azldev.md.tmpl | 8 + .../content/withoutlockfile/azldev.md.tmpl | 8 + .../app/azldev/cmds/component/component.go | 1 + .../app/azldev/cmds/component/distgitrepo.go | 265 +++++++++++++++++ .../azldev/cmds/component/distgitrepo_test.go | 276 ++++++++++++++++++ 10 files changed, 700 insertions(+) create mode 100644 docs/user/how-to/extract-dist-git-repo.md create mode 100644 docs/user/reference/cli/azldev_component_dist-git-repo.md create mode 100644 internal/app/azldev/cmds/component/distgitrepo.go create mode 100644 internal/app/azldev/cmds/component/distgitrepo_test.go diff --git a/docs/user/README.md b/docs/user/README.md index fe38aa55d..1d341cd90 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -5,6 +5,7 @@ - [Create a Project](./how-to/create-project.md) — set up a new Azure Linux project - [Add a Component](./how-to/add-component.md) — import or create a new package - [Build a Component](./how-to/build-component.md) — build RPMs from component definitions +- [Extract a Dist-Git Repository](./how-to/extract-dist-git-repo.md) — export a component's committed files and history - [Build an Image](./how-to/build-image.md) — build and boot Azure Linux images - [Set Up AI Coding Agents](./how-to/set-up-ai-agents.md) — emit agent skill and instruction files - [Preview the Lock-File-Free Mode](./how-to/preview-without-lockfile.md) — select lockless components diff --git a/docs/user/how-to/extract-dist-git-repo.md b/docs/user/how-to/extract-dist-git-repo.md new file mode 100644 index 000000000..61a737032 --- /dev/null +++ b/docs/user/how-to/extract-dist-git-repo.md @@ -0,0 +1,65 @@ +# How To: Extract a Component's Dist-Git Repository + +Use `azldev comp dist-git-repo` to extract one component's committed rendered +files and their Git history from the local project into a standalone repository. +This works for both locked and lockless components. + +Install **git-filter-repo** first. If `git filter-repo --version` is unavailable, +the command fails with an installation message before replacing any output. + +## Extract a component + +Specify exactly one component name and a required output directory: + +```bash +azldev comp dist-git-repo curl -o ./dist-git/curl +``` + +`--output-dir` is the long form of `-o`. Relative output paths are relative to +your current working directory. To select a project elsewhere, use the global +`-C` option: + +```bash +azldev -C /path/to/project comp dist-git-repo curl -o ./dist-git/curl +``` + +The command clones the local project repository's current branch using +`git clone --single-branch --no-local`. It then runs +`git filter-repo --subdirectory-filter` in the clone, selecting: + +```text +// +``` + +The path passed to the filter is relative to the project repository's Git root. +For example, `project.rendered-specs-dir = "specs"` selects `specs/c/curl`; +`SLOF` selects `specs/s/SLOF`. A configured nested rendered-specs directory is +respected. + +The selected files become the standalone repository's root: + +```bash +git -C ./dist-git/curl log --oneline +ls ./dist-git/curl +``` + +Only committed content is exported. This command does not render files, refresh pins, +or fetch the component's upstream Fedora/dist-git repository. Render and commit +changes first if you want them included. + +## Replace an existing output + +Even an empty existing directory is rejected unless you provide `--force`: + +```bash +azldev comp dist-git-repo curl --output-dir ./dist-git/curl --force +``` + +**`--force` deletes the existing output directory and its contents.** Prerequisite, +source-path, and output-safety checks run before deletion. The source +repository and home directory (including their ancestors), Git metadata, and paths +containing symbolic links are not allowed as destinations. + +If cloning or filtering fails, any partial output remains for inspection. +Resolve the cause and retry with `--force`. A global `--dry-run` makes no +filesystem changes and does not run Git commands. diff --git a/docs/user/reference/cli/azldev_component.md b/docs/user/reference/cli/azldev_component.md index 9fb435e1b..e46d7a5db 100644 --- a/docs/user/reference/cli/azldev_component.md +++ b/docs/user/reference/cli/azldev_component.md @@ -44,6 +44,7 @@ components defined in the project configuration. * [azldev component build](azldev_component_build.md) - Build packages for components * [azldev component changed](azldev_component_changed.md) - Detect which components changed between two git refs * [azldev component diff-sources](azldev_component_diff-sources.md) - Show the diff that overlays apply to a component's sources +* [azldev component dist-git-repo](azldev_component_dist-git-repo.md) - Extract a component's committed dist-git history into a repository * [azldev component history](azldev_component_history.md) - Report per-component change activity and customization detail * [azldev component list](azldev_component_list.md) - List components in this project * [azldev component make-lockless](azldev_component_make-lockless.md) - Select the lockless workflow and refresh upstream commits diff --git a/docs/user/reference/cli/azldev_component_dist-git-repo.md b/docs/user/reference/cli/azldev_component_dist-git-repo.md new file mode 100644 index 000000000..d9639ac53 --- /dev/null +++ b/docs/user/reference/cli/azldev_component_dist-git-repo.md @@ -0,0 +1,64 @@ + + +## azldev component dist-git-repo + +Extract a component's committed dist-git history into a repository + +### Synopsis + +Clone the local project repository's current branch with '--single-branch' +and '--no-local', then run 'git filter-repo --subdirectory-filter' to retain +only //. +The component's dist-git files become the new repository's root. + +Requires exactly one component name, '--output-dir' (-o), and git-filter-repo. +Only committed files and history are exported; this does not render the component +or clone its upstream distro repository. + +An existing output directory is rejected unless '--force' is supplied. +With '--force', that directory is deleted and recreated after preflight checks. +For safety, the source repository and home directory (including their ancestors), +Git metadata, and paths containing symbolic links cannot be used as output directories. +Clone or filtering failures leave any partial output available for inspection. + +``` +azldev component dist-git-repo [flags] +``` + +### Examples + +``` + azldev comp dist-git-repo curl -o ./dist-git/curl + azldev comp dist-git-repo rust-asyncband --output-dir ./dist-git/rust-asyncband --force +``` + +### Options + +``` + --force delete and recreate an existing output directory + -h, --help help for dist-git-repo + -o, --output-dir string destination for the extracted repository (required) +``` + +### Options inherited from parent commands + +``` + -y, --accept-all accept all prompts + --color mode output colorization mode {always, auto, never} (default auto) + --concurrency int base concurrency limit (default: number of logical CPUs; minimum 1) + --config-file stringArray additional TOML config file(s) to merge (may be repeated) + -n, --dry-run dry run only (do not take action) + --network-retries int maximum number of attempts for network operations (minimum 1) (default 3) + --no-default-config disable default configuration + -O, --output-format fmt output format {csv, json, markdown, table} (default table) + --permissive-config do not fail on unknown fields in TOML config files + -C, --project string path to Azure Linux project + -q, --quiet only enable minimal output + --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) + -v, --verbose enable verbose output +``` + +### SEE ALSO + +* [azldev component](azldev_component.md) - Manage components + diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 86289a919..2488c1bb5 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -172,6 +172,17 @@ func TestComponentTomlSkillDocumentsPerComponentRenderModes(t *testing.T) { } } +func TestSkillsDocumentDistGitRepo(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "azldev comp dist-git-repo -o ") + assert.Contains(t, doc, "--single-branch --no-local") + assert.Contains(t, doc, "specs//") + assert.Contains(t, doc, "and git-filter-repo") + } +} + func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { for _, withoutLockfile := range []bool{false, true} { doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev", testParams()) diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index cbc72232e..f895f7de2 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -102,6 +102,7 @@ Everyday tasks (add `-q -O json` when scripting): | Add a component | `azldev comp add` | | Build a component | `azldev comp build -p ` | | Render specs | `azldev comp render -p ` (or `-a`) | +| Extract committed dist-git history | `azldev comp dist-git-repo -o ` | | Refresh a lock | `azldev comp update -p ` | | Refresh a lockless component | `azldev comp refresh-upstream-commit -p ` | | List / build images | `azldev image list` / `azldev image build` | @@ -109,6 +110,13 @@ Everyday tasks (add `-q -O json` when scripting): `comp` is an alias for `component`. Always confirm current syntax with `azldev --help`. +`comp dist-git-repo` requires exactly one component name, `--output-dir` (`-o`), +and git-filter-repo. It clones the local project's current branch with +`--single-branch --no-local`, then filters +`{{ .RenderedSpecsDir }}//` into the repository root. +Only committed rendered files/history are exported; it does not fetch upstream. +Existing output requires `--force`, which deletes it after preflight checks. + The hidden `advanced` group (`adv`) contains specialist integrations such as MCP and mock helpers; it is intentionally omitted from normal help. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 20ae46a70..fa68c02ff 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -100,12 +100,20 @@ Everyday tasks (add `-q -O json` when scripting): | Add a component | `azldev comp add` | | Build a component | `azldev comp build -p ` | | Render specs | `azldev comp render -p ` (or `-a`) | +| Extract committed dist-git history | `azldev comp dist-git-repo -o ` | | Refresh an upstream commit | `azldev comp refresh-upstream-commit -p ` | | List / build images | `azldev image list` / `azldev image build` | `comp` is an alias for `component`. Always confirm current syntax with `azldev --help`. +`comp dist-git-repo` requires exactly one component name, `--output-dir` (`-o`), +and git-filter-repo. It clones the local project's current branch with +`--single-branch --no-local`, then filters +`{{ .RenderedSpecsDir }}//` into the repository root. +Only committed rendered files/history are exported; it does not fetch upstream. +Existing output requires `--force`, which deletes it after preflight checks. + The hidden `advanced` group (`adv`) contains specialist integrations such as MCP and mock helpers; it is intentionally omitted from normal help. diff --git a/internal/app/azldev/cmds/component/component.go b/internal/app/azldev/cmds/component/component.go index 549700739..87024e752 100644 --- a/internal/app/azldev/cmds/component/component.go +++ b/internal/app/azldev/cmds/component/component.go @@ -31,6 +31,7 @@ components defined in the project configuration.`, buildOnAppInit(app, cmd) changedOnAppInit(app, cmd) diffSourcesOnAppInit(app, cmd) + distGitRepoOnAppInit(app, cmd) listOnAppInit(app, cmd) prepareOnAppInit(app, cmd) renderOnAppInit(app, cmd) diff --git a/internal/app/azldev/cmds/component/distgitrepo.go b/internal/app/azldev/cmds/component/distgitrepo.go new file mode 100644 index 000000000..9d16753b3 --- /dev/null +++ b/internal/app/azldev/cmds/component/distgitrepo.go @@ -0,0 +1,265 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" + "github.com/spf13/afero" + "github.com/spf13/cobra" +) + +type DistGitRepoOptions struct { + Component string + OutputDir string + Force bool +} + +type DistGitRepoResult struct { + Component string `json:"component"` + OutputDir string `json:"outputDir"` +} + +func distGitRepoOnAppInit(_ *azldev.App, parent *cobra.Command) { + parent.AddCommand(NewDistGitRepoCmd()) +} + +// NewDistGitRepoCmd constructs the "component dist-git-repo" command. +func NewDistGitRepoCmd() *cobra.Command { + var options DistGitRepoOptions + + cmd := &cobra.Command{ + Use: "dist-git-repo ", + Short: "Extract a component's committed dist-git history into a repository", + Long: `Clone the local project repository's current branch with '--single-branch' +and '--no-local', then run 'git filter-repo --subdirectory-filter' to retain +only //. +The component's dist-git files become the new repository's root. + +Requires exactly one component name, '--output-dir' (-o), and git-filter-repo. +Only committed files and history are exported; this does not render the component +or clone its upstream distro repository. + +An existing output directory is rejected unless '--force' is supplied. +With '--force', that directory is deleted and recreated after preflight checks. +For safety, the source repository and home directory (including their ancestors), +Git metadata, and paths containing symbolic links cannot be used as output directories. +Clone or filtering failures leave any partial output available for inspection.`, + Example: ` azldev comp dist-git-repo curl -o ./dist-git/curl + azldev comp dist-git-repo rust-asyncband --output-dir ./dist-git/rust-asyncband --force`, + Args: cobra.ExactArgs(1), + RunE: azldev.RunFuncWithExtraArgs(func(env *azldev.Env, args []string) (interface{}, error) { + options.Component = args[0] + + return CreateComponentDistGitRepo(env, &options) + }), + ValidArgsFunction: components.GenerateComponentNameCompletions, + } + cmd.Flags().StringVarP(&options.OutputDir, "output-dir", "o", "", + "destination for the extracted repository (required)") + _ = cmd.MarkFlagRequired("output-dir") + _ = cmd.MarkFlagDirname("output-dir") + cmd.Flags().BoolVar(&options.Force, "force", false, "delete and recreate an existing output directory") + + return cmd +} + +// CreateComponentDistGitRepo exports a component's committed rendered subtree and its history. +func CreateComponentDistGitRepo(env *azldev.Env, options *DistGitRepoOptions) (*DistGitRepoResult, error) { + if options.OutputDir == "" { + return nil, errors.New("'--output-dir' (-o) is required") + } + + if err := fileutils.ValidateFilename(options.Component); err != nil { + return nil, fmt.Errorf("invalid component name:\n%w", err) + } + + resolver := components.NewResolver(env) + + resolver.SuppressLockWarnings = true + if _, err := resolver.GetComponentByName(options.Component); err != nil { + return nil, fmt.Errorf("resolving component:\n%w", err) + } + + output, err := distGitRepoOutputPath(env, options.OutputDir) + if err != nil { + return nil, err + } + + exists, err := fileutils.Exists(env.FS(), output) + if err != nil { + return nil, fmt.Errorf("checking output directory %#q:\n%w", output, err) + } + + if exists && !options.Force { + return nil, fmt.Errorf("output directory %#q already exists; use '--force' to replace it", output) + } + + result := &DistGitRepoResult{Component: options.Component, OutputDir: output} + if env.DryRun() { + slog.Info("Dry run; would extract component dist-git repository", "component", options.Component, "outputDir", output) + + return result, nil + } + + if err := requireGitFilterRepo(env); err != nil { + return nil, err + } + + repoRoot, err := git.RunInDir(env, env, env.ProjectDir(), "rev-parse", "--show-toplevel") + if err != nil { + return nil, fmt.Errorf("finding project repository root:\n%w", err) + } + + if err := validateDistGitRepoOutput(env, repoRoot, output); err != nil { + return nil, err + } + + subdir, err := distGitRepoSubdirectory(env, repoRoot, options.Component) + if err != nil { + return nil, err + } + + return createDistGitRepo(env, repoRoot, subdir, output, exists, result) +} + +func requireGitFilterRepo(env *azldev.Env) error { + prereq := &prereqs.PackagePrereq{VersionCheck: func(ctx opctx.Ctx) error { + if _, err := git.RunInDir(ctx, ctx, env.ProjectDir(), "filter-repo", "--version"); err != nil { + return fmt.Errorf("git-filter-repo must be installed to use the dist-git-repo command:\n%w", err) + } + + return nil + }} + if err := prereqs.RequireExecutable(env, "git", prereq); err != nil { + return fmt.Errorf("checking dist-git-repo prerequisites:\n%w", err) + } + + return nil +} + +func distGitRepoOutputPath(env *azldev.Env, output string) (string, error) { + if !filepath.IsAbs(output) { + cwd, err := env.OSEnv().Getwd() + if err != nil { + return "", fmt.Errorf("getting current directory:\n%w", err) + } + + output = filepath.Join(cwd, output) + } + + return filepath.Clean(output), nil +} + +func validateDistGitRepoOutput(env *azldev.Env, repoRoot, output string) error { + if _, err := repoRelPath(output, repoRoot); err == nil { + return fmt.Errorf("output directory %#q must not contain the source repository", output) + } + + if home := env.OSEnv().Getenv("HOME"); home != "" { + if _, err := repoRelPath(output, home); err == nil { + return fmt.Errorf("output directory %#q must not contain the home directory", output) + } + } + + if _, err := repoRelPath(filepath.Join(repoRoot, ".git"), output); err == nil { + return fmt.Errorf("output directory %#q must not be inside source Git metadata", output) + } + + gitDir, err := git.RunInDir(env, env, env.ProjectDir(), "rev-parse", "--git-common-dir") + if err != nil { + return fmt.Errorf("finding source Git metadata:\n%w", err) + } + + if !filepath.IsAbs(gitDir) { + gitDir = filepath.Join(env.ProjectDir(), gitDir) + } + + if _, err := repoRelPath(output, gitDir); err == nil { + return fmt.Errorf("output directory %#q must not contain source Git metadata", output) + } + + if _, err := repoRelPath(gitDir, output); err == nil { + return fmt.Errorf("output directory %#q must not be inside source Git metadata", output) + } + + return validateDistGitRepoOutputLinks(env, output) +} + +func validateDistGitRepoOutputLinks(env *azldev.Env, output string) error { + lstater, ok := env.FS().(afero.Lstater) + if !ok { + return errors.New("cannot check output directory for symbolic links") + } + + for path := output; ; path = filepath.Dir(path) { + info, _, err := lstater.LstatIfPossible(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("checking output path %#q:\n%w", path, err) + } + + if err == nil && info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("output path %#q must not contain symbolic links", path) + } + + if path == filepath.Dir(path) { + break + } + } + + return nil +} + +func distGitRepoSubdirectory(env *azldev.Env, repoRoot, name string) (string, error) { + specDir := env.Config().Project.RenderedSpecsDir + if specDir == "" { + specDir = projectconfig.DefaultRenderedSpecsDir + } + + subdir, err := projectSpecRepoRelPath(env, repoRoot, filepath.Join(specDir, strings.ToLower(name[:1]), name)) + if err != nil { + return "", err + } + + subdir = filepath.ToSlash(subdir) + + return subdir, nil +} + +func createDistGitRepo( + env *azldev.Env, repoRoot, subdir, output string, exists bool, result *DistGitRepoResult, +) (*DistGitRepoResult, error) { + if exists { + if err := env.FS().RemoveAll(output); err != nil { + return nil, fmt.Errorf("removing output directory %#q:\n%w", output, err) + } + } + + if err := fileutils.MkdirAll(env.FS(), filepath.Dir(output)); err != nil { + return nil, fmt.Errorf("creating output parent directory:\n%w", err) + } + + if _, err := git.RunInDir(env, env, repoRoot, + "clone", "--single-branch", "--no-local", "--", repoRoot, output); err != nil { + return nil, fmt.Errorf("cloning project repository:\n%w", err) + } + + if _, err := git.RunInDir(env, env, output, "filter-repo", "--subdirectory-filter", subdir); err != nil { + return nil, fmt.Errorf("filtering component dist-git history:\n%w", err) + } + + return result, nil +} diff --git a/internal/app/azldev/cmds/component/distgitrepo_test.go b/internal/app/azldev/cmds/component/distgitrepo_test.go new file mode 100644 index 000000000..343798065 --- /dev/null +++ b/internal/app/azldev/cmds/component/distgitrepo_test.go @@ -0,0 +1,276 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component_test + +import ( + "errors" + "os/exec" + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + componentcmds "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + distGitTestOutput = "/exports/component" + distGitTestProject = "/project" + distGitTestRevParse = "rev-parse" + distGitTestCommonDir = "--git-common-dir" + distGitTestClone = "clone" +) + +func newDistGitRepoTestEnv(t *testing.T, name string) *testutils.TestEnv { + t.Helper() + + env := testutils.NewTestEnv(t) + env.Config.Components[name] = projectconfig.ComponentConfig{Name: name} + env.Config.Project.RenderedSpecsDir = "/project/specs" + env.CmdFactory.RegisterCommandInSearchPath("git") + + return env +} + +func TestDistGitRepoCmd_Arguments(t *testing.T) { + for _, args := range [][]string{ + {}, + {"curl", "vim", "-o", distGitTestOutput}, + {"curl"}, + {"curl", "-o", ""}, + {"unknown-component", "-o", distGitTestOutput}, + {"cu*", "-o", distGitTestOutput}, + } { + t.Run(filepath.Join(args...), func(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected command: %v", cmd.Args) + + return "", nil + } + cmd := componentcmds.NewDistGitRepoCmd() + cmd.SetArgs(args) + require.Error(t, cmd.ExecuteContext(env.Env)) + }) + } +} + +func TestCreateComponentDistGitRepo(t *testing.T) { + for _, testCase := range []struct { + name, component, specDir, wantSubdir string + force bool + }{ + {name: "default path", component: "curl", wantSubdir: "specs/c/curl"}, + {name: "custom relative path", component: "curl", specDir: "build/rendered", wantSubdir: "build/rendered/c/curl"}, + {name: "uppercase component", component: "SLOF", specDir: "/project/rendered", wantSubdir: "rendered/s/SLOF"}, + {name: "replace existing directory", component: "curl", wantSubdir: "specs/c/curl", force: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := newDistGitRepoTestEnv(t, testCase.component) + + env.Config.Project.RenderedSpecsDir = testCase.specDir + if testCase.force { + require.NoError(t, fileutils.MkdirAll(env.TestFS, distGitTestOutput)) + require.NoError(t, fileutils.WriteFile(env.TestFS, distGitTestOutput+"/stale", []byte("keep"), + fileperms.PublicFile)) + } + + var commands [][]string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + commands = append(commands, cmd.Args) + switch cmd.Args[3] { + case distGitTestRevParse: + if cmd.Args[4] == distGitTestCommonDir { + return "/project/.git", nil + } + + return distGitTestProject, nil + case distGitTestClone: + exists, err := fileutils.Exists(env.TestFS, distGitTestOutput) + require.NoError(t, err) + assert.False(t, exists, "clone must receive a fresh destination") + require.NoError(t, fileutils.MkdirAll(env.TestFS, distGitTestOutput)) + } + + return "", nil + } + result, err := componentcmds.CreateComponentDistGitRepo(env.Env, &componentcmds.DistGitRepoOptions{ + Component: testCase.component, OutputDir: distGitTestOutput, Force: testCase.force, + }) + require.NoError(t, err) + assert.Equal(t, testCase.component, result.Component) + assert.Equal(t, distGitTestOutput, result.OutputDir) + assert.Equal(t, [][]string{ + {"git", "-C", distGitTestProject, "filter-repo", "--version"}, + {"git", "-C", distGitTestProject, distGitTestRevParse, "--show-toplevel"}, + {"git", "-C", distGitTestProject, distGitTestRevParse, distGitTestCommonDir}, + { + "git", "-C", distGitTestProject, distGitTestClone, "--single-branch", "--no-local", "--", + distGitTestProject, distGitTestOutput, + }, + {"git", "-C", distGitTestOutput, "filter-repo", "--subdirectory-filter", testCase.wantSubdir}, + }, commands) + }) + } +} + +func TestCreateComponentDistGitRepo_ExistingOutput(t *testing.T) { + for _, nonempty := range []bool{false, true} { + t.Run(map[bool]string{false: "empty", true: "nonempty"}[nonempty], func(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + require.NoError(t, fileutils.MkdirAll(env.TestFS, distGitTestOutput)) + + if nonempty { + require.NoError(t, fileutils.WriteFile(env.TestFS, distGitTestOutput+"/keep", []byte("keep"), + fileperms.PublicFile)) + } + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected command: %v", cmd.Args) + + return "", nil + } + _, err := componentcmds.CreateComponentDistGitRepo(env.Env, &componentcmds.DistGitRepoOptions{ + Component: "curl", OutputDir: distGitTestOutput, + }) + require.ErrorContains(t, err, "already exists") + require.ErrorContains(t, err, "--force") + exists, err := fileutils.DirExists(env.TestFS, distGitTestOutput) + require.NoError(t, err) + assert.True(t, exists) + }) + } +} + +func TestCreateComponentDistGitRepo_FailuresPreserveOutput(t *testing.T) { + for _, step := range []string{"filter-repo", distGitTestRevParse} { + t.Run(step, func(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + require.NoError(t, fileutils.MkdirAll(env.TestFS, distGitTestOutput)) + + const sentinel = distGitTestOutput + "/keep" + require.NoError(t, fileutils.WriteFile(env.TestFS, sentinel, []byte("original"), fileperms.PublicFile)) + + failure := errors.New("preflight failed") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.NotEqual(t, distGitTestClone, cmd.Args[3]) + + if cmd.Args[3] == step { + return "", failure + } + + return distGitTestProject, nil + } + _, err := componentcmds.CreateComponentDistGitRepo(env.Env, &componentcmds.DistGitRepoOptions{ + Component: "curl", OutputDir: distGitTestOutput, Force: true, + }) + require.ErrorIs(t, err, failure) + + if step == "filter-repo" { + require.ErrorContains(t, err, "git-filter-repo must be installed to use the dist-git-repo command") + } + + content, err := fileutils.ReadFile(env.TestFS, sentinel) + require.NoError(t, err) + assert.Equal(t, "original", string(content)) + }) + } +} + +func TestCreateComponentDistGitRepo_UnsafeOutput(t *testing.T) { + for _, output := range []string{ + "/", "/project", "/project/..", "/project/.git", "/project/.git/objects", + "/shared", "/shared/git", "/shared/git/objects", + } { + t.Run(output, func(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.NotEqual(t, distGitTestClone, cmd.Args[3]) + + if cmd.Args[3] == distGitTestRevParse { + if cmd.Args[4] == distGitTestCommonDir { + return "/shared/git", nil + } + + return distGitTestProject, nil + } + + return "", nil + } + _, err := componentcmds.CreateComponentDistGitRepo(env.Env, &componentcmds.DistGitRepoOptions{ + Component: "curl", OutputDir: output, Force: true, + }) + require.ErrorContains(t, err, "output directory") + exists, err := fileutils.Exists(env.TestFS, "/project/azldev.toml") + require.NoError(t, err) + assert.True(t, exists, "source project must remain intact") + }) + } +} + +func TestCreateComponentDistGitRepo_CommandFailures(t *testing.T) { + for _, step := range []string{distGitTestClone, "filter"} { + t.Run(step, func(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + failure := errors.New("export failed") + + var cloned bool + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + switch cmd.Args[3] { + case distGitTestRevParse: + if cmd.Args[4] == distGitTestCommonDir { + return "/project/.git", nil + } + + return distGitTestProject, nil + case distGitTestClone: + if step == distGitTestClone { + return "", failure + } + + cloned = true + case "filter-repo": + if cmd.Args[4] == "--subdirectory-filter" { + assert.True(t, cloned) + + return "", failure + } + } + + return "", nil + } + _, err := componentcmds.CreateComponentDistGitRepo(env.Env, &componentcmds.DistGitRepoOptions{ + Component: "curl", OutputDir: distGitTestOutput, + }) + require.ErrorIs(t, err, failure) + }) + } +} + +func TestCreateComponentDistGitRepo_DryRun(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + dryEnv := azldev.NewEnv(t.Context(), azldev.EnvOptions{ + ProjectDir: "/project", Config: env.Config, Interfaces: env.TestInterfaces, + EventListener: env.EventListener, DryRunnable: azldev.NewAppDryRunnable(true), + }) + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected command: %v", cmd.Args) + + return "", nil + } + _, err := componentcmds.CreateComponentDistGitRepo(dryEnv, &componentcmds.DistGitRepoOptions{ + Component: "curl", OutputDir: distGitTestOutput, + }) + require.NoError(t, err) + exists, err := fileutils.Exists(env.TestFS, distGitTestOutput) + require.NoError(t, err) + assert.False(t, exists) +} From 585944923eeefebe21beefe106aee8beffb2754c Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 6 Nov 2026 16:54:47 +0000 Subject: [PATCH 38/70] fix(component): align local lockless rendering behavior Require a clean repository for every lockless component and apply the unchanged-render refusal, timestamp rebuilds, and automatic commits to local components. Keep local Release and changelog unchanged, including when --rpmdev-bumpspec is supplied. Exclude TOML files defining the selected local spec path during render copying while retaining other sidecars and configured source manifests. Preserve locked rendering behavior and update user and agent guidance. Verify with Mage build, unit tests, full checks, documentation generation, and native local-render tests using azurelinux-release inputs. Container scenarios remain blocked by host Docker provider initialization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 41 ++++--- .../reference/cli/azldev_component_render.md | 20 +-- .../app/azldev/agentskill/agentskill_test.go | 10 +- .../azldev/agentskill/content/azldev.md.tmpl | 13 +- .../content/build-component.md.tmpl | 2 +- .../agentskill/content/comp-toml.md.tmpl | 5 + .../content/refresh-upstream-commit.md.tmpl | 8 +- .../content/withoutlockfile/azldev.md.tmpl | 13 +- .../withoutlockfile/build-component.md.tmpl | 2 +- .../content/withoutlockfile/comp-toml.md.tmpl | 7 +- internal/app/azldev/cmds/component/render.go | 32 +++-- .../azldev/cmds/component/render_lockless.go | 53 +++++--- .../render_lockless_internal_test.go | 116 +++++++++++++++++- .../azldev/cmds/component/render_message.go | 2 +- .../component/render_modes_internal_test.go | 9 +- .../app/azldev/core/sources/sourceprep.go | 38 ++++-- .../sourceproviders/localcontentsprovider.go | 58 ++++++++- .../localcontentsprovider_test.go | 38 ++++++ .../sourceproviders/sourcemanager.go | 10 +- .../sourceproviders/sourcemanager_test.go | 10 +- 20 files changed, 393 insertions(+), 94 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index f793ee7e8..58662188c 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -133,17 +133,22 @@ Commit the refreshed TOML before rendering: rendering compares the upstream pins from committed configuration. Lockless rendering neither generates synthetic history nor expands `%autorelease` or `%autochangelog`. -## Render and Commit an Upstream Component - -The following process applies only to **upstream components selected as -lockless**. Local components skip the upstream release/changelog carryover and -automatic commit steps. All lockless rendering runs on the host without mock; +## Render and Commit a Component + +The following process applies to **all components selected as lockless**. +Local components skip upstream-history queries and release/changelog carryover; +their Release and changelog are never automatically modified, including with +`--rpmdev-bumpspec`. They copy local files except the TOML defining the local spec +path, retain other TOML sidecars, and participate in unchanged-render checks and +automatic commits. For local components, `--changelog-comment` supplies only the +Git commit message; without it, Git opens its own editor. +All lockless rendering runs on the host without mock; components using lock files retain their existing mock-based rendering. Before any upstream checkout or component preparation, rendering checks the entire project Git repository for staged changes, unstaged tracked changes, and untracked files. Git-ignored files do not block rendering. If any selected -component is lockless and upstream, a dirty repository stops the whole invocation. +component is lockless, a dirty repository stops the whole invocation. Commit or remove those changes first, or explicitly bypass this check: ```sh @@ -154,9 +159,10 @@ azldev comp render -p --allow-uncommitted `--allow-uncommitted` is specified. The override bypasses only the cleanliness check: carryover and upstream-change comparison still use committed history, and failure recovery can discard uncommitted files in the component output -directory. Locked and local components do not trigger this check. +directory. Locked components do not trigger this check. -The renderer clones only the branch selected by the component's effective +For upstream components, the renderer clones only the branch selected by the +component's effective `spec.upstream-distro` (or the project default) into temporary storage. The resolved distro version's `dist-git-branch` supplies the branch name. The clone retains that branch's full history, not every upstream branch, so pinned commits @@ -258,7 +264,7 @@ and any top-level `.no-change-rebuild` on both sides. `--check-only` stops here, before the final release bump. If normal rendering detects no changes, it returns an error indicating there has been no change since the last render, without replacing the existing output -or writing a failure marker. To explicitly rebuild unchanged upstream content, +or writing a failure marker. To explicitly rebuild unchanged content, pass `--no-change-rebuild`, a boolean switch taking no argument: ```bash @@ -267,13 +273,14 @@ azldev comp render -p curl --no-change-rebuild For an unchanged render with this flag, the tool runs host `date -Is` and writes its exact output to `.no-change-rebuild` at the working dist-git root before -bumping. The marker is then published and committed with the rendered files. +bumping upstream specs. Local specs are not bumped. The marker is then published +and committed with the rendered files. The flag does not create a marker when comparison detects real changes. The root marker is ignored in both the working upstream checkout and existing output, even if its contents differ. Nested files with the same name are compared. Changed renders, or explicitly requested no-change rebuilds, then invoke host -`rpmdev-bumpspec -c` **only for non-autorelease specs**, with the exact +`rpmdev-bumpspec -c` **only for upstream non-autorelease specs**, with the exact `--changelog-comment` value. There is no default comment. If the flag is omitted, rendering opens `$EDITOR` (including configured arguments), or a local editor (`editor`, `sensible-editor`, `vi`, or `nano`), to obtain the entry. The editor @@ -287,7 +294,8 @@ An edited message containing only ignored lines or editor failure aborts renderi before output replacement. Missing-comment finishing is serialized to prevent simultaneous editors. -Autorelease specs skip `rpmdev-bumpspec` entirely. With `--changelog-comment`, +Local components and autorelease specs skip `rpmdev-bumpspec` entirely. +With `--changelog-comment`, Git uses the supplied message; without it, Git opens its own configured editor to obtain the commit message, using a template containing upstream changes. `--no-git-commit` skips that Git prompt, but non-autorelease specs still need a @@ -308,7 +316,8 @@ Release edits preserve the existing spaces and tabs between `Release:` and its value, including during overlays and carryover, so aligned spec headers stay aligned with either spec editor. Rendered output retains all prepared dist-git files except `.git`, including -unreferenced sidecars, hidden files, and subdirectories. +unreferenced sidecars, hidden files, and subdirectories. Local spec-defining TOML +files are excluded during copying. Lockless rendering does not run `spectool` or filter files; `render.skip-file-filter` has no effect on lockless components. Synthetic history and its legacy release adjustment are skipped. The spec is **not** preprocessed with `rpmautospec process-distgit`: @@ -333,8 +342,8 @@ include only their own dist-git output. Unchanged Git content does not produce an empty commit. Pass `--no-git-commit` to skip automatic staging and committing while still rendering and bumping non-autorelease specs. The rendered output remains available for review -and manual commit. Both flags affect only lockless upstream components; -locked and local rendering retain their existing behavior. +and manual commit. These flags affect all lockless components; +locked rendering retains its existing behavior. Local Release/changelog are not bumped. ```bash azldev comp render -p curl --changelog-comment "Fix curl patches" --no-git-commit @@ -370,7 +379,7 @@ required by the renderer with `--no-git-commit`. performs the final `rpmdev-bumpspec` invocation, replaces rendered files, or creates project commits. `--check-only` also never creates a rebuild timestamp, even with `--no-change-rebuild`. The no-change refusal and rebuild flag apply only -to lockless upstream components; local and locked components retain their existing +to all lockless components; locked components retain their existing behavior. Change reporting includes a forced rebuild marker and changes made by the final bump. diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 2501c03ff..966a8ee53 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -44,6 +44,13 @@ rpmautospec generate-changelog/calculate-release are used only when needed for release history or provenance. Provenance is captured before carryover/bumping and reused for the macros file. Locked components keep their existing behavior. +Local lockless components copy local files except the TOML defining the spec path. +Their Release and changelog are never automatically modified, even with --rpmdev-bumpspec. +They also reject unchanged renders unless --no-change-rebuild is provided, and +automatically commit rendered output unless --no-git-commit is provided. +For local components, --changelog-comment supplies only the Git commit message; +without it, Git prompts in its own editor. Output must be inside the project repository. + When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to any component in the project config). Per-component dirs that ARE in config @@ -54,11 +61,10 @@ preserved. When using a custom output directory (--output-dir), --force is required alongside --clean-stale as a safety measure. This flag is only valid with -a. -Before rendering any lockless upstream component, require a clean Git repository. +Before rendering any lockless component, require a clean Git repository. Uncommitted tracked changes and untracked files anywhere in the repository block rendering; Git-ignored files are excluded. --allow-uncommitted bypasses this check. ---check-only and --no-git-commit do not bypass it. Locked and local components are -unaffected. +--check-only and --no-git-commit do not bypass it. Locked components are unaffected. ``` azldev component render [flags] @@ -81,8 +87,8 @@ azldev component render [flags] ``` -a, --all-components Include all components - --allow-uncommitted allow lockless upstream rendering with uncommitted or untracked files in the project repository - --changelog-comment string comment passed verbatim to rpmdev-bumpspec -c and used as the Git commit message for lockless upstream components (if omitted, prompt in an editor) + --allow-uncommitted allow lockless rendering with uncommitted or untracked files in the project repository + --changelog-comment string Git commit message for lockless components and verbatim rpmdev-bumpspec -c comment for upstream non-autorelease specs (if omitted, prompt in an editor) --check-only render to a staging area and compare against the existing on-disk output without modifying the output directory. Exits 0 when nothing would change and 1 when any component would drift. With -a + --clean-stale, also fails on orphan rendered-spec directories. Intended for CI gates. --clean-stale prune rendered-spec directories that no longer correspond to a configured component (only with -a; requires -f with -o). Top-level non-component siblings are preserved. -p, --component stringArray Component name pattern @@ -90,8 +96,8 @@ azldev component render [flags] --fail-on-error exit with error if any component fails to render (useful for CI) -f, --force allow overwriting existing rendered component directories -h, --help help for render - --no-change-rebuild allow unchanged lockless upstream renders by recording 'date -Is' output in '.no-change-rebuild' - --no-git-commit leave lockless upstream rendered output uncommitted without staging it + --no-change-rebuild allow unchanged lockless renders by recording 'date -Is' output in '.no-change-rebuild' + --no-git-commit leave lockless rendered output uncommitted without staging it -o, --output-dir string output directory for rendered specs (overrides rendered-specs-dir from config) --rpmdev-bumpspec Use rpmdev-bumpspec instead of the legacy static release calculation --skip-lock-validation skip lock file consistency checks diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 2488c1bb5..4a842ab1f 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -169,6 +169,9 @@ func TestComponentTomlSkillDocumentsPerComponentRenderModes(t *testing.T) { assert.Contains(t, doc, "for every command") assert.Contains(t, doc, "two blank lines and `#`-commented instructions") assert.Contains(t, doc, "Blank lines and lines starting with `#` are removed, with no trailing newline") + assert.Contains(t, doc, "Local lockless components skip upstream carryover; Release and changelog remain unchanged") + assert.Contains(t, doc, "TOML defining the local spec path") + assert.Contains(t, doc, "participate in unchanged-render checks and automatic commits") } } @@ -200,7 +203,10 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "Version detection and carryover use the upstream spec before overlays") assert.Contains(t, doc, "Overlays can override the prepared release and changelog") assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") - assert.Contains(t, doc, "Unchanged lockless upstream renders require `--no-change-rebuild`") + assert.Contains(t, doc, "Unchanged lockless renders require `--no-change-rebuild`") + assert.Contains(t, doc, "Local lockless components skip upstream carryover; Release and changelog remain unchanged") + assert.Contains(t, doc, "TOML defining the local spec path") + assert.Contains(t, doc, "participate in unchanged-render checks and automatic commits") assert.Contains(t, doc, "host `date -Is` output") assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") @@ -249,7 +255,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "Version detection and carryover use the upstream spec before overlays") assert.Contains(t, doc, "Overlays can override the prepared release and changelog") assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") - assert.Contains(t, doc, "Unchanged lockless upstream renders require `--no-change-rebuild`") + assert.Contains(t, doc, "Unchanged lockless renders require `--no-change-rebuild`") assert.Contains(t, doc, "host `date -Is` output") assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index f895f7de2..bad48b8a7 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -42,13 +42,13 @@ then runs the refresh pipeline. The list is written in lexical order, one name per line without blank lines or duplicates. Existing names and lock files are retained. Already-listed components are still refreshed; a refresh failure leaves the list updated. -Before preparing any selected lockless upstream component, rendering requires a +Before preparing any selected lockless component, rendering requires a clean project Git repository: staged changes, unstaged tracked changes, and untracked files anywhere in the repository block the whole invocation. Git-ignored files are excluded. Use `--allow-uncommitted` for intentional dirty renders; `--check-only` and `--no-git-commit` do not bypass this check. The override does not change committed-history carryover or failure recovery. -Locked and local components do not trigger this check. +Only locked components do not trigger this check. For listed upstream components, rendering prepares the changelog before Release, then writes macros, applies overlays, and updates `sources`. @@ -58,7 +58,7 @@ Non-autorelease rendering then runs host `rpmdev-bumpspec`, without synthetic hi release adjustment. Lockless rendering preserves the existing spaces and tabs after `Release:` when editing its value, and compares output before the final release bump; `--check-only` stops before that bump. -Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors +Unchanged lockless renders require `--no-change-rebuild`; otherwise rendering errors without replacing existing output. The flag records host `date -Is` output in `.no-change-rebuild` before bumping, only when no changes were detected. Comparison ignores this root marker in both existing output and the working upstream checkout. @@ -86,8 +86,11 @@ bumps non-autorelease specs. Upstream changes remain in the Git commit body, not the changelog comment. Commit refreshed TOML before rendering. Configure Git author identity and install `rpmdevtools`. `--check-only` never creates project commits. -Local components skip upstream carryover and automatic commits; locked -components retain their existing render behavior. +Local lockless components skip upstream carryover; Release and changelog remain unchanged. +They exclude the TOML defining the local spec path, while retaining other TOML sidecars. +They participate in unchanged-render checks and automatic commits. +For local components, `--changelog-comment` supplies only the Git message; otherwise +Git opens its own editor. Locked components retain their existing render behavior. Top-level commands in this build: diff --git a/internal/app/azldev/agentskill/content/build-component.md.tmpl b/internal/app/azldev/agentskill/content/build-component.md.tmpl index 42d00ccc8..8f917a47b 100644 --- a/internal/app/azldev/agentskill/content/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/build-component.md.tmpl @@ -38,7 +38,7 @@ Prefer `comp render` for quick overlay verification; use `comp diff-sources` to exact overlay effect (it fetches sources once, applies overlays to a copy, and diffs the two trees). Builds can be slow — set generous timeouts. -For lockless upstream components, uncommitted tracked changes or untracked files +For all lockless components, uncommitted tracked changes or untracked files anywhere in the project repository require `comp render --allow-uncommitted`. Git-ignored files are excluded; the override does not change committed-history carryover or failure recovery. Commit inputs before the final render (see the diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index c7ccc42d4..0116ea618 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -106,6 +106,11 @@ rendered output directory; later commits contain only their component output. ## Render configuration +Local lockless components skip upstream carryover; Release and changelog remain unchanged. +They exclude the TOML defining the local spec path, retain other TOML sidecars, +and participate in unchanged-render checks and automatic commits. +For local components, `--changelog-comment` supplies only the Git message. + azldev reads `lockless-components.txt` beside the top-level `azldev.toml`. Listed component names use the lockless workflow for every command; all others use lock files. A missing or empty list means all components use diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 153e47e9d..0c55274d8 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -60,7 +60,11 @@ Git-ignored files are excluded. Use `--allow-uncommitted` for intentional dirty renders; `--check-only` and `--no-git-commit` do not bypass this check. The override does not change committed-history carryover or failure recovery, which can discard uncommitted files in the component output directory. -Locked and local components do not trigger this check. +Only locked components do not trigger this check. +Local lockless components also require a clean repository, exclude the TOML defining +the local spec path, and participate in unchanged-render checks and automatic commits. +Their Release and changelog remain unchanged; `--changelog-comment` supplies only +their Git message. ```sh git add \ @@ -95,7 +99,7 @@ Version detection and carryover use the upstream spec before overlays. Overlays can override the prepared release and changelog. Carryover is not repeated after overlays, even if they change the version. Rendering compares output before the final release bump; `--check-only` stops before that bump. -Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors +Unchanged lockless renders require `--no-change-rebuild`; otherwise rendering errors without replacing existing output. The flag records host `date -Is` output in `.no-change-rebuild` before bumping, only when no changes were detected. Comparison ignores this root marker in both existing output and the working upstream checkout. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index fa68c02ff..df1e3ae35 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -39,13 +39,12 @@ The list is written in lexical order, one name per line without blank lines or d Existing names and lock files are retained. Already-listed components are still refreshed; a refresh failure leaves the list updated. -Before preparing any upstream component, rendering requires a clean project Git +Before preparing any lockless component, rendering requires a clean project Git repository: staged changes, unstaged tracked changes, and untracked files anywhere in the repository block the whole invocation. Git-ignored files are excluded. Use `--allow-uncommitted` for intentional dirty renders; `--check-only` and `--no-git-commit` do not bypass this check. The override does not change -committed-history carryover or failure recovery. Local components do not trigger -this check. +committed-history carryover or failure recovery. For upstream components, rendering prepares the changelog before Release, then writes macros, applies overlays, and updates `sources`. @@ -54,7 +53,7 @@ Overlays can override the prepared release and changelog. Non-autorelease rendering then runs host `rpmdev-bumpspec`. Release edits preserve the existing spaces and tabs after `Release:`. Rendering compares output before the final release bump; `--check-only` stops before that bump. -Unchanged lockless upstream renders require `--no-change-rebuild`; otherwise rendering errors +Unchanged lockless renders require `--no-change-rebuild`; otherwise rendering errors without replacing existing output. The flag records host `date -Is` output in `.no-change-rebuild` before bumping, only when no changes were detected. Comparison ignores this root marker in both existing output and the working upstream checkout. @@ -84,6 +83,12 @@ Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. Lockless rendering retains all prepared dist-git files except `.git`, without file filtering or `spectool` queries. Synthetic history and its legacy release adjustment are skipped during rendering. + +Local lockless components skip upstream carryover; Release and changelog remain unchanged. +They exclude the TOML defining the local spec path, while retaining other TOML sidecars. +They participate in unchanged-render checks and automatic commits. +For local components, `--changelog-comment` supplies only the Git message; otherwise +Git opens its own editor. `--check-only` never creates project commits. Local components skip upstream carryover and automatic commits. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl index 46bb3be38..2aa731b31 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl @@ -41,7 +41,7 @@ two trees). Builds can be slow — set generous timeouts. Finalize with `azldev comp refresh-upstream-commit -p ` before opening a PR (see the `azldev-refresh-upstream-commit` skill). -For upstream components, uncommitted tracked changes or untracked files anywhere +For all lockless components, uncommitted tracked changes or untracked files anywhere in the project repository require `comp render --allow-uncommitted`. Git-ignored files are excluded; the override does not change committed-history carryover or failure recovery. Commit inputs before the final render. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 4b4c01dca..5d50f9212 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -98,8 +98,11 @@ needed, and configure Git author identity. Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded and the `changelog` sidecar is preserved. Synthetic history and its legacy release adjustment are skipped during rendering. -`--check-only` never creates project commits. Local components skip upstream -carryover and automatic commits. +`--check-only` never creates project commits. +Local lockless components skip upstream carryover; Release and changelog remain unchanged. +They exclude the TOML defining the local spec path, retain other TOML sidecars, +and participate in unchanged-render checks and automatic commits. +For local components, `--changelog-comment` supplies only the Git message. The first automatic commit also includes pending repository changes outside the rendered output directory; later commits contain only their component output. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index a7bc2e1ed..ac280c229 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -50,11 +50,17 @@ func renderOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { parentCmd.AddCommand(NewRenderCmd()) } -const renderWorktreeHelp = `Before rendering any lockless upstream component, require a clean Git repository. +const renderWorktreeHelp = `Before rendering any lockless component, require a clean Git repository. Uncommitted tracked changes and untracked files anywhere in the repository block rendering; Git-ignored files are excluded. --allow-uncommitted bypasses this check. ---check-only and --no-git-commit do not bypass it. Locked and local components are -unaffected.` +--check-only and --no-git-commit do not bypass it. Locked components are unaffected.` + +const renderLocalHelp = `Local lockless components copy local files except the TOML defining the spec path. +Their Release and changelog are never automatically modified, even with --rpmdev-bumpspec. +They also reject unchanged renders unless --no-change-rebuild is provided, and +automatically commit rendered output unless --no-git-commit is provided. +For local components, --changelog-comment supplies only the Git commit message; +without it, Git prompts in its own editor. Output must be inside the project repository.` // NewRenderCmd constructs a [cobra.Command] for the "component render" CLI subcommand. func NewRenderCmd() *cobra.Command { @@ -101,7 +107,7 @@ The spec retains its RPM macros, including %autorelease and %autochangelog. All prepared files except .git are retained, without filtering or spectool. Host rpmautospec generate-changelog/calculate-release are used only when needed for release history or provenance. Provenance is captured before carryover/bumping -and reused for the macros file. Locked components keep their existing behavior. +and reused for the macros file. Locked components keep their existing behavior.` + "\n\n" + renderLocalHelp + ` When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to @@ -161,14 +167,14 @@ func addRenderOptions(cmd *cobra.Command, options *RenderOptions) { "and 1 when any component would drift. With -a + --clean-stale, also fails "+ "on orphan rendered-spec directories. Intended for CI gates.") cmd.Flags().BoolVar(&options.NoChangeRebuild, "no-change-rebuild", false, - "allow unchanged lockless upstream renders by recording 'date -Is' output in '.no-change-rebuild'") + "allow unchanged lockless renders by recording 'date -Is' output in '.no-change-rebuild'") cmd.Flags().StringVar(&options.ChangelogComment, "changelog-comment", "", - "comment passed verbatim to rpmdev-bumpspec -c and used as the Git commit message "+ - "for lockless upstream components (if omitted, prompt in an editor)") + "Git commit message for lockless components and verbatim rpmdev-bumpspec -c comment "+ + "for upstream non-autorelease specs (if omitted, prompt in an editor)") cmd.Flags().BoolVar(&options.NoGitCommit, "no-git-commit", false, - "leave lockless upstream rendered output uncommitted without staging it") + "leave lockless rendered output uncommitted without staging it") cmd.Flags().BoolVar(&options.AllowUncommitted, "allow-uncommitted", false, - "allow lockless upstream rendering with uncommitted or untracked files in the project repository") + "allow lockless rendering with uncommitted or untracked files in the project repository") addRPMDevBumpspecFlag(cmd, &options.RPMDevBumpspec) // --check-only is a read-only diff against on-disk state; --fail-on-error @@ -610,7 +616,7 @@ func prepareComponentSources( sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), ) if lockless { - preparerOpts = append(preparerOpts, sources.WithoutSyntheticHistory(), + preparerOpts = append(preparerOpts, sources.WithoutSyntheticHistory(), sources.WithSkipLocalSpecConfig(), sources.WithAutoreleaseResolver(hostRenderProcessor{env: env})) } else { preparerOpts = append(preparerOpts, sources.WithMockProcessor(mockProcessor)) @@ -618,18 +624,18 @@ func prepareComponentSources( var update *locklessRenderUpdate - if lockless && comp.GetConfig().Spec.SourceType == projectconfig.SpecSourceTypeUpstream { + if lockless { preparerOpts = append(preparerOpts, sources.WithFetchedSources( func(_ context.Context, fetched components.Component, dir string) error { var prepErr error //nolint:contextcheck // The callback context is the captured preparation env. - update, prepErr = prepareLocklessRender(env, fetched, dir, compOutputDir, specEditor) + update, prepErr = prepareFetchedLocklessRender(env, fetched, dir, compOutputDir, specEditor) return prepErr })) } - if rpmDevBumpspec { + if rpmDevBumpspec && (!lockless || comp.GetConfig().Spec.SourceType != projectconfig.SpecSourceTypeLocal) { preparerOpts = append(preparerOpts, sources.WithRPMDevBumpspec(env, env.WorkDir(), "")) } diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index d6c7c721b..8cf7a5a2b 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -42,6 +42,7 @@ type locklessRenderUpdate struct { changes string noGitCommit bool subjectProvided bool + local bool } type lastRender struct { @@ -75,8 +76,7 @@ func validateRenderComponents( } for _, comp := range comps { - if !env.ComponentWithoutLockfile(comp.GetName()) || - comp.GetConfig().Spec.SourceType != projectconfig.SpecSourceTypeUpstream { + if !env.ComponentWithoutLockfile(comp.GetName()) { continue } @@ -101,26 +101,16 @@ func prepareLocklessRender( env *azldev.Env, comp components.Component, dir, outputDir string, editor spec.EditorMode, ) (*locklessRenderUpdate, error) { - repo, root, err := openRenderRepo(env) + repo, update, err := prepareLocklessRenderOutput(env, outputDir) if err != nil { return nil, err } - path, err := repoRelPath(root, outputDir) - if err != nil { - return nil, fmt.Errorf("lockless render output must be inside the project repository:\n%w", err) - } - - history, err := readLastRender(env, repo, root, path, comp.GetName()) + history, err := readLastRender(env, repo, update.repoRoot, update.path, comp.GetName()) if err != nil { return nil, fmt.Errorf("reading committed render state:\n%w", err) } - update := &locklessRenderUpdate{ - repoRoot: root, - path: path, - } - _, opened, err := openLocklessSpec(env, dir, comp.GetName(), editor) if err != nil { return nil, err @@ -142,6 +132,37 @@ func prepareLocklessRender( return update, nil } +func prepareFetchedLocklessRender( + env *azldev.Env, comp components.Component, dir, outputDir string, editor spec.EditorMode, +) (*locklessRenderUpdate, error) { + if comp.GetConfig().Spec.SourceType != projectconfig.SpecSourceTypeLocal { + return prepareLocklessRender(env, comp, dir, outputDir, editor) + } + + _, update, err := prepareLocklessRenderOutput(env, outputDir) + if update != nil { + update.local = true + } + + return update, err +} + +func prepareLocklessRenderOutput( + env *azldev.Env, outputDir string, +) (repo *gogit.Repository, update *locklessRenderUpdate, err error) { + repo, root, err := openRenderRepo(env) + if err != nil { + return nil, nil, err + } + + path, err := repoRelPath(root, outputDir) + if err != nil { + return nil, nil, fmt.Errorf("lockless render output must be inside the project repository:\n%w", err) + } + + return repo, &locklessRenderUpdate{repoRoot: root, path: path}, nil +} + func readLastRender( env *azldev.Env, repo *gogit.Repository, root, path, name string, ) (lastRender, error) { @@ -819,6 +840,10 @@ func finishLocklessSpec( } } + if update.local { + return !changed, nil + } + bumped, err := bumpLocklessSpec(env, path, update) return !changed || bumped, err diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index d046396b7..069da4c94 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -33,7 +33,10 @@ import ( "go.uber.org/mock/gomock" ) -const renderTestRevParse = "rev-parse" +const ( + renderTestRevParse = "rev-parse" + renderTestProjectRoot = "/project" +) func renderTestSpec(version, release, changelog string) string { return fmt.Sprintf("Name: curl\nVersion: %s\nRelease: %s\nSummary: Test\nLicense: MIT\n"+ @@ -1116,14 +1119,13 @@ func TestWithAutoreleaseBase(t *testing.T) { } } -func TestPrepareComponentSources_LocalAndLockedSkipNewLifecycle(t *testing.T) { +func TestPrepareComponentSources_LockedSkipNewLifecycle(t *testing.T) { for _, testCase := range []struct { name string source projectconfig.SpecSourceType lockless bool }{ {name: "locked local", source: projectconfig.SpecSourceTypeLocal}, - {name: "lockless local", source: projectconfig.SpecSourceTypeLocal, lockless: true}, {name: "locked upstream", source: projectconfig.SpecSourceTypeUpstream}, } { t.Run(testCase.name, func(t *testing.T) { @@ -1148,8 +1150,6 @@ func TestPrepareComponentSources_LocalAndLockedSkipNewLifecycle(t *testing.T) { t.Cleanup(func() { newSourcePreparer = original }) - // A successful fetch without a spec deliberately stops at overlays. - // The new lifecycle would instead require a project Git repository. _, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/rendered", spec.EditorLegacy, false, testCase.lockless) require.ErrorContains(t, err, "failed to apply overlays") @@ -1157,6 +1157,105 @@ func TestPrepareComponentSources_LocalAndLockedSkipNewLifecycle(t *testing.T) { } } +func TestPrepareComponentSources_LocalLocklessKeepsSpecAndSetsCommitState(t *testing.T) { + env := testutils.NewTestEnv(t) + ctrl := gomock.NewController(t) + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + }) + + repo, _ := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{"README": []byte("local project")}}}) + originalRepo := openRenderRepo + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { return repo, renderTestProjectRoot, nil } + + t.Cleanup(func() { openRenderRepo = originalRepo }) + + manager := sourceproviders_test.NewMockSourceManager(ctrl) + manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile(env.TestFS, filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("1", "7%{?dist}", "* Keep local changelog\n")), fileperms.PublicFile) + }) + + originalPreparer := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + t.Cleanup(func() { newSourcePreparer = originalPreparer }) + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected upstream history or release query: %v", cmd.Args) + + return "", nil + } + prep, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/project/SPECS/c/curl", + spec.EditorLegacy, true, true) + require.NoError(t, err) + require.NotNil(t, prep.update) + assert.True(t, prep.update.local) + assert.Equal(t, renderTestProjectRoot, prep.update.repoRoot) + assert.Equal(t, "SPECS/c/curl", prep.update.path) + + content, err := fileutils.ReadFile(env.TestFS, "/staging/curl/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: 7%{?dist}") + assert.Contains(t, string(content), "* Keep local changelog\n") +} + +func TestFinishLocalLocklessSpec_DoesNotBump(t *testing.T) { + for _, testCase := range []struct { + name string + changed, rebuild bool + wantError bool + }{ + {name: "changed", changed: true}, + {name: "unchanged", wantError: true}, + {name: "explicit rebuild", rebuild: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RegisterCommandInSearchPath("date") + + path := "/working/curl.spec" + content := renderTestSpec("1", "7%{?dist}", "* Keep local changelog\n") + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(content), fileperms.PublicFile)) + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{"date", "-Is"}, cmd.Args) + + return "2026-11-06T16:00:00+00:00\n", nil + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("local specs must never invoke bumpspec or a changelog editor: %v", cmd.Args) + + return nil + } + + changed, err := finishLocklessSpec(env.Env, path, &locklessRenderUpdate{local: true}, + testCase.changed, testCase.rebuild) + if testCase.wantError { + require.ErrorIs(t, err, errNoRenderChanges) + } else { + require.NoError(t, err) + assert.Equal(t, testCase.rebuild, changed) + } + + got, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Equal(t, content, string(got)) + + exists, err := fileutils.Exists(env.TestFS, "/working/.no-change-rebuild") + require.NoError(t, err) + assert.Equal(t, testCase.rebuild, exists) + }) + } +} + func TestPrepareComponentSources_ChangelogAndReleaseBeforeOverlays(t *testing.T) { const releaseTag = "Release: " @@ -1384,6 +1483,11 @@ func TestPrepareComponentSources_LocklessSkipsSyntheticHistory(t *testing.T) { for _, release := range []string{"7%{?dist}", "%autorelease -b 9"} { t.Run(release, func(t *testing.T) { env := testutils.NewTestEnv(t) + repo, _ := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{"README": []byte("local")}}}) + originalRepo := openRenderRepo + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { return repo, renderTestProjectRoot, nil } + + t.Cleanup(func() { openRenderRepo = originalRepo }) ctrl := gomock.NewController(t) comp := components_testutils.NewMockComponent(ctrl) comp.EXPECT().GetName().AnyTimes().Return("curl") @@ -1431,7 +1535,7 @@ func TestPrepareComponentSources_LocklessSkipsSyntheticHistory(t *testing.T) { return "", nil } - _, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/rendered", + _, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/project/rendered", spec.EditorLegacy, false, true) require.NoError(t, err) content, err := fileutils.ReadFile(env.TestFS, "/staging/curl/curl.spec") diff --git a/internal/app/azldev/cmds/component/render_message.go b/internal/app/azldev/cmds/component/render_message.go index 428d62c34..954da6014 100644 --- a/internal/app/azldev/cmds/component/render_message.go +++ b/internal/app/azldev/cmds/component/render_message.go @@ -26,7 +26,7 @@ const renderChangelogTemplate = "\n\n" + func renderFinishConcurrency(env *azldev.Env, prepared []*preparedComponent, checkOnly bool) int { if !checkOnly && !env.DryRun() { for _, prep := range prepared { - if prep.update != nil && !prep.update.subjectProvided && prep.update.subject == "" { + if prep.update != nil && !prep.update.local && !prep.update.subjectProvided && prep.update.subject == "" { return 1 } } diff --git a/internal/app/azldev/cmds/component/render_modes_internal_test.go b/internal/app/azldev/cmds/component/render_modes_internal_test.go index fe634cced..3da11203e 100644 --- a/internal/app/azldev/cmds/component/render_modes_internal_test.go +++ b/internal/app/azldev/cmds/component/render_modes_internal_test.go @@ -90,7 +90,7 @@ func (p *renderModeObservingPreparer) PrepareSources( return p.err } -func TestRenderComponentsRequiresCleanLocklessUpstreamRepository(t *testing.T) { +func TestRenderComponentsRequiresCleanLocklessRepository(t *testing.T) { statusErr := errors.New("git status failed") for _, testCase := range []struct { @@ -197,7 +197,7 @@ func TestLocklessRenderWorktreeCheckScope(t *testing.T) { }{ {name: "locked upstream", source: projectconfig.SpecSourceTypeUpstream}, {name: "locked local", source: projectconfig.SpecSourceTypeLocal}, - {name: "lockless local", source: projectconfig.SpecSourceTypeLocal, lockless: true}, + {name: "lockless local", source: projectconfig.SpecSourceTypeLocal, lockless: true, checked: true}, {name: "lockless upstream", source: projectconfig.SpecSourceTypeUpstream, lockless: true, checked: true}, } { t.Run(testCase.name, func(t *testing.T) { @@ -238,7 +238,10 @@ func TestLocklessRenderPreservesAllFilesWithoutMock(t *testing.T) { Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, } env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { - t.Fatalf("unexpected query during lockless rendering: %v", cmd.Args) + assert.Equal(t, []string{ + "git", "-C", "/project", "status", "--porcelain=v1", + "--untracked-files=all", "--ignore-submodules=none", + }, cmd.Args) return "", nil } diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 4c789bf04..8bc000777 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -148,6 +148,11 @@ func WithSkipLookaside() PreparerOption { } } +// WithSkipLocalSpecConfig excludes the TOML defining the local spec path from fetched files. +func WithSkipLocalSpecConfig() PreparerOption { + return func(p *sourcePreparerImpl) { p.skipLocalSpecConfig = true } +} + // WithFetchedSources runs prepare on the fetched checkout before overlays and // synthetic history are applied. func WithFetchedSources(prepare func(context.Context, components.Component, string) error) PreparerOption { @@ -223,6 +228,8 @@ type sourcePreparerImpl struct { // source preparation. Git-tracked files are still fetched. skipLookaside bool + skipLocalSpecConfig bool + // cmdFactory is used to shell out to git for fingerprint change detection // in the project repository. Set via [WithGitRepo]. cmdFactory opctx.CmdFactory @@ -328,20 +335,8 @@ func (p *sourcePreparerImpl) PrepareSources( } } - // Preserve the upstream .git directory only when dist-git creation is - // requested via --with-git. This is required so that overlay commits can be - // appended on top of the upstream commit log during synthetic history generation. - var fetchOpts []sourceproviders.FetchComponentOption - if applyOverlays && p.withGitRepo { - fetchOpts = append(fetchOpts, sourceproviders.WithPreserveGitDir()) - } - - if p.skipLookaside { - fetchOpts = append(fetchOpts, sourceproviders.WithSkipLookaside()) - } - // Fetch the component first (spec, sidecar files, and upstream source tarballs). - err := p.sourceManager.FetchComponent(ctx, component, outputDir, fetchOpts...) + err := p.sourceManager.FetchComponent(ctx, component, outputDir, p.componentFetchOptions(applyOverlays)...) if err != nil { return fmt.Errorf("failed to fetch sources for component %#q:\n%w", component.GetName(), err) @@ -392,6 +387,23 @@ func (p *sourcePreparerImpl) PrepareSources( return nil } +func (p *sourcePreparerImpl) componentFetchOptions(applyOverlays bool) []sourceproviders.FetchComponentOption { + var options []sourceproviders.FetchComponentOption + if applyOverlays && p.withGitRepo { + options = append(options, sourceproviders.WithPreserveGitDir()) + } + + if p.skipLookaside { + options = append(options, sourceproviders.WithSkipLookaside()) + } + + if p.skipLocalSpecConfig { + options = append(options, sourceproviders.WithSkipLocalSpecConfig()) + } + + return options +} + func (p *sourcePreparerImpl) prepareFetched( ctx context.Context, component components.Component, dir string, ) error { diff --git a/internal/providers/sourceproviders/localcontentsprovider.go b/internal/providers/sourceproviders/localcontentsprovider.go index e17db5b5d..16b8931d9 100644 --- a/internal/providers/sourceproviders/localcontentsprovider.go +++ b/internal/providers/sourceproviders/localcontentsprovider.go @@ -15,6 +15,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/pelletier/go-toml/v2" "github.com/spf13/afero" ) @@ -26,6 +27,7 @@ func FetchLocalComponent( dryRunnable opctx.DryRunnable, eventListener opctx.EventListener, fileSystem opctx.FS, component components.Component, destDirPath string, resolveRequiredFiles bool, + options ...FetchComponentOption, ) error { if dryRunnable == nil { return errors.New("dry runnable cannot be nil") @@ -57,7 +59,9 @@ func FetchLocalComponent( } // Copy all files from the source directory to the destination directory - err = copySourceDirectory(dryRunnable, fileSystem, sourceDirPath, destDirPath) + opts := resolveFetchComponentOptions(options) + + err = copySourceDirectory(dryRunnable, fileSystem, sourceDirPath, destDirPath, component, opts.SkipLocalSpecConfig) if err != nil { return fmt.Errorf("failed to copy source directory for component %#q:\n%w", component.GetName(), err) @@ -90,13 +94,27 @@ func acquireSourceDirPath(component components.Component) (string, error) { // copySourceDirectory copies all files from the source directory to the destination directory, // preserving file mode bits (e.g., executable bits for scripts). -func copySourceDirectory(dryRunnable opctx.DryRunnable, fs opctx.FS, sourceDirPath, destDirPath string) error { +func copySourceDirectory( + dryRunnable opctx.DryRunnable, fs opctx.FS, sourceDirPath, destDirPath string, + component components.Component, skipLocalSpecConfig bool, +) error { copyOptions := fileutils.CopyDirOptions{ CopyFileOptions: fileutils.CopyFileOptions{ PreserveFileMode: true, }, } + if skipLocalSpecConfig { + copyOptions.FileFilter = func(_ opctx.FS, destPath string) (bool, error) { + relative, err := filepath.Rel(destDirPath, destPath) + if err != nil { + return false, fmt.Errorf("resolving copied local file path:\n%w", err) + } + + return shouldCopyLocalSource(fs, component, filepath.Join(sourceDirPath, relative)) + } + } + err := fileutils.CopyDirRecursive(dryRunnable, fs, sourceDirPath, destDirPath, copyOptions) if err != nil { return fmt.Errorf("failed to copy files from %#q to %#q:\n%w", @@ -106,6 +124,42 @@ func copySourceDirectory(dryRunnable opctx.DryRunnable, fs opctx.FS, sourceDirPa return nil } +func shouldCopyLocalSource(fs opctx.FS, component components.Component, sourcePath string) (bool, error) { + if filepath.Ext(sourcePath) != ".toml" { + return true, nil + } + + content, err := fileutils.ReadFile(fs, sourcePath) + if err != nil { + return false, fmt.Errorf("reading local TOML file %#q:\n%w", sourcePath, err) + } + + var ( + config projectconfig.ConfigFile + localPath string + ) + + // Unrelated TOML sidecars need not parse as azldev configuration. + if toml.Unmarshal(content, &config) == nil { + localPath = config.Components[component.GetName()].Spec.Path + } + + if localPath == "" { + return true, nil + } + + if !filepath.IsAbs(localPath) { + localPath = filepath.Join(filepath.Dir(sourcePath), localPath) + } + + specPath, err := component.GetSpec().GetPath() + if err != nil { + return false, fmt.Errorf("getting local spec path:\n%w", err) + } + + return filepath.Clean(localPath) != filepath.Clean(specPath), nil +} + // copyRequiredFiles resolves and copies any required files that may be stored separately // from the spec file's directory. func copyRequiredFiles( diff --git a/internal/providers/sourceproviders/localcontentsprovider_test.go b/internal/providers/sourceproviders/localcontentsprovider_test.go index 46c860806..132ee14dd 100644 --- a/internal/providers/sourceproviders/localcontentsprovider_test.go +++ b/internal/providers/sourceproviders/localcontentsprovider_test.go @@ -134,6 +134,44 @@ func TestFetchLocalComponent_NoRequiredFiles(t *testing.T) { require.Equal(t, testParams.specContents, string(contents)) } +func TestFetchLocalComponent_SkipDefiningSpecConfig(t *testing.T) { + for _, skip := range []bool{false, true} { + t.Run(map[bool]string{false: "ordinary fetch", true: "lockless render"}[skip], func(t *testing.T) { + params := setupLocalFetcherTest(t) + + files := map[string]string{ + "definition.toml": "[components.test.spec]\ntype = 'local'\npath = 'test.spec'\n", + "metadata.toml": "[components.test]\n", + "settings.toml": "setting = true\n", + "other.toml": "[components.other.spec]\ntype = 'local'\npath = 'test.spec'\n", + "overridden.toml": "[components.test.spec]\ntype = 'local'\npath = 'other.spec'\n", + "invalid.toml": "not valid TOML [\n", + "nested/def.toml": "[components.test.spec]\ntype = 'local'\npath = '../test.spec'\n", + } + for name, content := range files { + require.NoError(t, fileutils.WriteFile(params.env.TestFS, filepath.Join(params.specDir, name), + []byte(content), fileperms.PublicFile)) + } + + var options []sourceproviders.FetchComponentOption + if skip { + options = append(options, sourceproviders.WithSkipLocalSpecConfig()) + } + + require.NoError(t, sourceproviders.FetchLocalComponent(params.env.DryRunnable, + params.env.EventListener, params.env.TestFS, params.component, testOutputDir, false, options...)) + + for name := range files { + exists, err := fileutils.Exists(params.env.TestFS, filepath.Join(testOutputDir, name)) + require.NoError(t, err) + + want := !skip || (name != "definition.toml" && name != "nested/def.toml") + require.Equal(t, want, exists, name) + } + }) + } +} + func TestFetchLocalComponent_RequiredFileMissing(t *testing.T) { testParams := setupLocalFetcherTest(t) diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index 4bf07d45a..93b861f7a 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -75,6 +75,8 @@ type FetchComponentOptions struct { // fetching. Git-tracked files (spec, patches, scripts) are still fetched from the // upstream clone. The sources manifest file remains available for validation. SkipLookaside bool + + SkipLocalSpecConfig bool } // FetchComponentOption is a functional option for configuring component fetch behavior. @@ -96,6 +98,11 @@ func WithSkipLookaside() FetchComponentOption { } } +// WithSkipLocalSpecConfig excludes the TOML defining a local spec's path from fetched files. +func WithSkipLocalSpecConfig() FetchComponentOption { + return func(o *FetchComponentOptions) { o.SkipLocalSpecConfig = true } +} + // resolveFetchComponentOptions applies all functional options and returns the resolved options. func resolveFetchComponentOptions(opts []FetchComponentOption) FetchComponentOptions { var resolved FetchComponentOptions @@ -705,7 +712,8 @@ func (m *sourceManager) resolveUpstreamSourceIdentity( func (m *sourceManager) fetchLocalComponent( ctx context.Context, component components.Component, destDirPath string, opts FetchComponentOptions, ) error { - err := FetchLocalComponent(m.dryRunnable, m.eventListener, m.fs, component, destDirPath, false) + err := FetchLocalComponent(m.dryRunnable, m.eventListener, m.fs, component, destDirPath, false, + func(o *FetchComponentOptions) { *o = opts }) if err != nil { return fmt.Errorf("failed to fetch local component %#q:\n%w", component.GetName(), err) diff --git a/internal/providers/sourceproviders/sourcemanager_test.go b/internal/providers/sourceproviders/sourcemanager_test.go index d7bf67d66..3e81fea09 100644 --- a/internal/providers/sourceproviders/sourcemanager_test.go +++ b/internal/providers/sourceproviders/sourcemanager_test.go @@ -522,12 +522,16 @@ func TestSourceManager_FetchComponent_LocalComponent_WithSkipLookaside(t *testin []byte("Name: test-component\nVersion: 1.0\n"), fileperms.PrivateFile)) require.NoError(t, fileutils.WriteFile(env.TestFS, specDir+"/sources", []byte("SHA512 (big-source.tar.gz) = abc123def456\n"), fileperms.PrivateFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, specDir+"/definition.toml", + []byte("[components.test-component.spec]\ntype = 'local'\npath = 'test-component.spec'\n"), + fileperms.PublicFile)) sourceManager, err := sourceproviders.NewSourceManager(env.Env, testDefaultDistro()) require.NoError(t, err) // With SkipLookaside, downloadLookasideSources is not called and the fetch succeeds. - err = sourceManager.FetchComponent(t.Context(), component, testDestDir, sourceproviders.WithSkipLookaside()) + err = sourceManager.FetchComponent(t.Context(), component, testDestDir, + sourceproviders.WithSkipLookaside(), sourceproviders.WithSkipLocalSpecConfig()) require.NoError(t, err) // Spec was copied to destination (FetchLocalComponent still ran). @@ -535,6 +539,10 @@ func TestSourceManager_FetchComponent_LocalComponent_WithSkipLookaside(t *testin require.NoError(t, err) assert.True(t, specExists, "spec file should exist in destination") + configExists, err := fileutils.Exists(env.TestFS, filepath.Join(testDestDir, "definition.toml")) + require.NoError(t, err) + assert.False(t, configExists, "defining local spec TOML must not be copied") + // Source tarball was NOT downloaded (lookaside was skipped). tarballExists, err := fileutils.Exists(env.TestFS, filepath.Join(testDestDir, "big-source.tar.gz")) require.NoError(t, err) From 3aaeb71e769e3d5fb835e3b546c23a21aec87ff5 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 6 Nov 2026 18:12:05 +0000 Subject: [PATCH 39/70] fix(component): collect lockless render messages before publication Collect required messages after comparison and before bumping or publishing output, using case-specific changelog and commit editor instructions. Skip unused prompts for autorelease and local renders with --no-git-commit. Prefill upstream changes with '- ' prefixes. Autorelease prompts put '...' on line two and changes on line three; static prompts start changes on line two. Reuse edited text without appending the original changes again. Reject blank or comment-only editor responses before output replacement. Commit noninteractively, preserving the first-line summary and body boundary. Local Release and changelog content remain untouched. Update user documentation, generated CLI help, emitted guidance, and tests. Unit tests, all quality checks, docs generation, and native smoke tests passed. Docker scenarios and snapshot updates remain blocked by host Docker-provider initialization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 48 ++-- .../reference/cli/azldev_component_render.md | 7 +- .../app/azldev/agentskill/agentskill_test.go | 10 + .../azldev/agentskill/content/azldev.md.tmpl | 15 +- .../agentskill/content/comp-toml.md.tmpl | 9 +- .../content/refresh-upstream-commit.md.tmpl | 16 +- .../content/withoutlockfile/azldev.md.tmpl | 18 +- .../content/withoutlockfile/comp-toml.md.tmpl | 11 +- internal/app/azldev/cmds/component/render.go | 11 +- .../azldev/cmds/component/render_lockless.go | 11 +- .../render_lockless_internal_test.go | 22 +- .../azldev/cmds/component/render_message.go | 136 +++++++--- .../component/render_message_internal_test.go | 239 +++++++++++++----- 13 files changed, 403 insertions(+), 150 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 58662188c..7b459adba 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -141,7 +141,8 @@ their Release and changelog are never automatically modified, including with `--rpmdev-bumpspec`. They copy local files except the TOML defining the local spec path, retain other TOML sidecars, and participate in unchanged-render checks and automatic commits. For local components, `--changelog-comment` supplies only the -Git commit message; without it, Git opens its own editor. +Git commit message; without it, rendering collects the message before replacing +output. Local components with `--no-git-commit` never prompt. All lockless rendering runs on the host without mock; components using lock files retain their existing mock-based rendering. @@ -282,26 +283,39 @@ output, even if its contents differ. Nested files with the same name are compare Changed renders, or explicitly requested no-change rebuilds, then invoke host `rpmdev-bumpspec -c` **only for upstream non-autorelease specs**, with the exact `--changelog-comment` value. There is no default comment. If the flag is omitted, -rendering opens `$EDITOR` (including configured arguments), or a local editor -(`editor`, `sensible-editor`, `vi`, or `nano`), to obtain the entry. The editor -starts with two blank lines followed by `#`-commented instructions. Save and close -the editor to continue. Blank and whitespace-only lines and lines starting with +rendering collects any needed message **after comparison and before bumping or +replacing output**, using `$EDITOR` (including configured arguments), or a local +editor (`editor`, `sensible-editor`, `vi`, or `nano`). The editor +starts with two blank lines followed by `#`-commented instructions when there are +no upstream changes. Otherwise, the first line is blank for your summary, and +each upstream-change line is prefilled with a `- ` prefix. Non-autorelease prompts +start those changes on line 2. Autorelease prompts put `...` on line 2 and start +the changes on line 3. These are editable message text, not instructional comments. +Save and close the editor to continue. Blank and whitespace-only lines and lines starting with `#` are removed; other message lines are preserved, with no trailing newline. The cleaned text is passed to `rpmdev-bumpspec -c` and reused as the Git commit -message, leaving a single blank line between the new and previous changelog -entries. Explicit `--changelog-comment` values remain unchanged. +message, separating its first line (the Git summary) from subsequent body lines +with a blank line. Bumpspec leaves a single blank line between the new and +previous changelog entries. Explicit `--changelog-comment` values remain unchanged. An edited message containing only ignored lines or editor failure aborts rendering before output replacement. Missing-comment finishing is serialized to prevent simultaneous editors. Local components and autorelease specs skip `rpmdev-bumpspec` entirely. -With `--changelog-comment`, -Git uses the supplied message; without it, Git opens its own configured editor -to obtain the commit message, using a template containing upstream changes. -`--no-git-commit` skips that Git prompt, but non-autorelease specs still need a -changelog entry. `--check-only` and dry runs never open editors. +When `--changelog-comment` is omitted, the prompt depends on how the message is used: + +| Component | Automatic Git commit | `--no-git-commit` | +|-----------|----------------------|-------------------| +| Upstream non-autorelease | Changelog and commit message | Changelog message only | +| Autorelease (upstream or local) | First-line changelog/commit summary, then commit body | No prompt | +| Local non-autorelease | Git commit message only; changelog stays unchanged | No prompt | + +Each case has its own instructional comments. The final Git commit always reuses +the supplied or already collected message; it never opens another editor. +`--check-only` and dry runs never open editors. For unattended rendering, provide `--changelog-comment`. -Upstream changes are not appended to the changelog entry. +Prefilled upstream changes are used only as retained in the edited message; +the original changes are not appended again to its changelog entry or Git body. ```bash azldev comp render -p curl --changelog-comment "Fix curl patches" @@ -330,9 +344,11 @@ existing output directory and moves all prepared entries except `.git` into its replacement. Files found only in the old output are not retained. Moves use filesystem renames, so the project work directory and rendered output must be on the same filesystem. -The renderer then commits the component output directory. The subject is -the supplied or edited changelog comment; autorelease without a comment uses -Git's interactive message editor. The body includes upstream changes, if any. +The renderer then commits the component output directory noninteractively, using +the supplied or already edited message. Its first line is the Git summary; +remaining lines form the body. Prefilled upstream changes appear only as retained +in the edited message; explicitly supplied flag messages keep the original +upstream changes in a separate Git body paragraph. The first successful automatic commit also includes all staged changes, unstaged tracked changes, and non-ignored untracked files outside `rendered-specs-dir` (or the active `--output-dir`). Other component diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 966a8ee53..17471aec1 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -31,7 +31,9 @@ upstream spec and last committed render before writing macros, applying overlays and updating sources. Overlays can override the prepared release and changelog. Compare output; unchanged renders require --no-change-rebuild to record a timestamp. Non-autorelease specs use rpmdev-bumpspec -c with --changelog-comment, or edited text. -Autorelease skips bumpspec; without --changelog-comment, Git prompts for a message. +Autorelease skips bumpspec. Missing messages are collected in an editor before +bumping or replacing output; the final Git commit never prompts. +Autorelease with --no-git-commit needs no message and never prompts. The first commit also includes all pending changes outside the rendered output directory. Later commits are component-only. --no-git-commit skips staging and committing. Output must be inside the project repository; configure Git author identity and @@ -49,7 +51,8 @@ Their Release and changelog are never automatically modified, even with --rpmdev They also reject unchanged renders unless --no-change-rebuild is provided, and automatically commit rendered output unless --no-git-commit is provided. For local components, --changelog-comment supplies only the Git commit message; -without it, Git prompts in its own editor. Output must be inside the project repository. +without it, rendering collects the message in an editor before replacing output. +--no-git-commit skips this prompt. Output must be inside the project repository. When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 4a842ab1f..11fc70e2f 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -172,6 +172,10 @@ func TestComponentTomlSkillDocumentsPerComponentRenderModes(t *testing.T) { assert.Contains(t, doc, "Local lockless components skip upstream carryover; Release and changelog remain unchanged") assert.Contains(t, doc, "TOML defining the local spec path") assert.Contains(t, doc, "participate in unchanged-render checks and automatic commits") + assert.Contains(t, doc, "The final Git commit never prompts") + assert.Contains(t, doc, "Autorelease with `--no-git-commit` never prompts") + assert.Contains(t, doc, "Blank/comment-only edited messages fail rendering") + assert.Contains(t, doc, "Each change line is prefixed with `- ` and is editable") } } @@ -214,6 +218,9 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "`--changelog-comment`") assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") assert.Contains(t, doc, "`$EDITOR` or a local editor") + assert.Contains(t, doc, "The final Git commit never prompts") + assert.Contains(t, doc, "Local components with `--no-git-commit` never prompt") + assert.Contains(t, doc, "Each change line is prefixed with `- ` and is editable") assert.Contains(t, doc, "two blank lines and `#`-commented instructions") assert.Contains(t, doc, "Blank lines and lines starting with `#` are removed, with no trailing newline") assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") @@ -263,6 +270,9 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "`--changelog-comment`") assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") assert.Contains(t, doc, "`$EDITOR` or a local editor") + assert.Contains(t, doc, "The final Git commit never prompts") + assert.Contains(t, doc, "Autorelease with `--no-git-commit` never prompts") + assert.Contains(t, doc, "Each change line is prefixed with `- ` and is editable") assert.Contains(t, doc, "two blank lines and `#`-commented instructions") assert.Contains(t, doc, "Blank lines and lines starting with `#` are removed, with no trailing newline") assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index bad48b8a7..dc2e9ad37 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -75,22 +75,27 @@ the rendered output directory. Other rendered paths are excluded even if staged; later commits include only their component's dist-git changes. `--changelog-comment` supplies the exact `rpmdev-bumpspec -c` value and commit message, with no default. If omitted, `$EDITOR` or a local editor -collects the entry using a template with two blank lines and `#`-commented instructions. +collects the entry using a template with two blank lines and `#`-commented instructions +when no upstream changes exist. Otherwise, line 1 is blank for the summary; +non-autorelease changes start on line 2, while autorelease puts `...` on line 2 +and starts changes on line 3. Each change line is prefixed with `- ` and is editable. Blank lines and lines starting with `#` are removed, with no trailing newline; the cleaned text is reused for the Git message. Explicit flag values remain unchanged. -Autorelease skips `rpmdev-bumpspec`; without a supplied comment, Git prompts -in its own editor for a message, with upstream changes in the template. +Autorelease skips `rpmdev-bumpspec`; missing messages are collected before output replacement. +Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. +The final Git commit never prompts; its first message line is the summary, and remaining lines are body. Provide `--changelog-comment` for unattended rendering. `--no-git-commit` skips automatic staging and committing, but still renders and bumps non-autorelease specs. -Upstream changes remain in the Git commit body, not the changelog comment. +Edited upstream changes are not appended again; explicit flag messages retain the upstream Git body. Commit refreshed TOML before rendering. Configure Git author identity and install `rpmdevtools`. `--check-only` never creates project commits. Local lockless components skip upstream carryover; Release and changelog remain unchanged. They exclude the TOML defining the local spec path, while retaining other TOML sidecars. They participate in unchanged-render checks and automatic commits. For local components, `--changelog-comment` supplies only the Git message; otherwise -Git opens its own editor. Locked components retain their existing render behavior. +rendering collects it before output replacement, with commit-only instructions for non-autorelease. +Local components with `--no-git-commit` never prompt. Locked components retain their existing render behavior. Top-level commands in this build: diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 0116ea618..db72264d4 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -95,8 +95,12 @@ flag, no bumpspec runtime discovery occurs and the legacy static calculation is used outside the lockless upstream render workflow. That workflow always runs host `rpmdev-bumpspec -c` only for non-autorelease specs and automatically commits the component output. Missing `--changelog-comment` opens `$EDITOR` or a local -editor for non-autorelease, or Git's own editor for an autorelease commit. -The changelog editor starts with two blank lines and `#`-commented instructions. +editor before output replacement, including for autorelease commits. +Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. +The final Git commit never prompts; its first message line is the summary, and remaining lines are body. +With no upstream changes, the editor starts with two blank lines and `#`-commented instructions. +Otherwise, non-autorelease changes start on line 2; autorelease puts `...` on line 2 +and starts changes on line 3. Each change line is prefixed with `- ` and is editable. Blank lines and lines starting with `#` are removed, with no trailing newline; the cleaned text is reused for the Git message. Explicit flag values remain unchanged. Lockless rendering skips @@ -110,6 +114,7 @@ Local lockless components skip upstream carryover; Release and changelog remain They exclude the TOML defining the local spec path, retain other TOML sidecars, and participate in unchanged-render checks and automatic commits. For local components, `--changelog-comment` supplies only the Git message. +Local components with `--no-git-commit` never prompt. azldev reads `lockless-components.txt` beside the top-level `azldev.toml`. Listed component names use the lockless workflow for every command; diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 0c55274d8..4fba9d47a 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -151,7 +151,7 @@ still carries an explicitly committed `-b 1` as-is. The final host `rpmdev-bumpspec -c` uses the exact `--changelog-comment` value or the cleaned editor-provided entry. There is no default comment. -Upstream changes are not appended to this comment. +Upstream changes are prefilled in the editor, not appended after editing. All prepared dist-git files except `.git` are retained, without file filtering or `spectool` queries. Synthetic history and its legacy release adjustment are skipped during lockless rendering. @@ -169,13 +169,17 @@ automatically commits that directory. The first commit also includes all staged changes, unstaged tracked changes, and non-ignored untracked files outside the rendered output directory. Other rendered paths are excluded even if staged; later commits include only their component's dist-git changes. -The message is the supplied changelog comment or, for non-autorelease specs, -the entry collected through `$EDITOR` or a local editor. There is no default. -The changelog editor starts with two blank lines and `#`-commented instructions. +The message is the supplied changelog comment or the entry collected through +`$EDITOR` or a local editor before output replacement. There is no default. +With no upstream changes, the editor starts with two blank lines and `#`-commented instructions. +Otherwise, non-autorelease changes start on line 2; autorelease puts `...` on line 2 +and starts changes on line 3. Each change line is prefixed with `- ` and is editable. Blank lines and lines starting with `#` are removed, with no trailing newline; the cleaned text is reused for the Git message. Explicit flag values remain unchanged. -Autorelease skips `rpmdev-bumpspec`; without a supplied comment, Git prompts -in its own editor for a message. The body/template contains upstream changes. +Autorelease skips `rpmdev-bumpspec`; missing messages are collected before output replacement. +Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. +The final Git commit never prompts; its first message line is the summary, and remaining lines are body. +Edited upstream changes are not appended again; explicit flag messages retain the upstream Git body. Provide `--changelog-comment` for unattended rendering. `--no-git-commit` skips automatic staging and committing, but still renders and bumps non-autorelease specs. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index df1e3ae35..dfccae214 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -68,15 +68,19 @@ Other rendered paths are excluded even if staged; later commits include only their component's dist-git changes. `--changelog-comment` supplies the exact `rpmdev-bumpspec -c` value and commit message, with no default. If omitted, `$EDITOR` or a local editor collects the -entry using a template with two blank lines and `#`-commented instructions. +entry using a template with two blank lines and `#`-commented instructions +when no upstream changes exist. Otherwise, line 1 is blank for the summary; +non-autorelease changes start on line 2, while autorelease puts `...` on line 2 +and starts changes on line 3. Each change line is prefixed with `- ` and is editable. Blank lines and lines starting with `#` are removed, with no trailing newline; the cleaned text is reused for the Git message. Explicit flag values remain unchanged. -Autorelease skips `rpmdev-bumpspec`; without a supplied comment, Git prompts -in its own editor for a message, with upstream changes in the template. +Autorelease skips `rpmdev-bumpspec`; missing messages are collected before output replacement. +Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. +The final Git commit never prompts; its first message line is the summary, and remaining lines are body. Provide `--changelog-comment` for unattended rendering. `--no-git-commit` skips automatic staging and committing, but still renders and bumps non-autorelease specs. -Upstream changes remain in the Git commit body, not the changelog comment. +Edited upstream changes are not appended again; explicit flag messages retain the upstream Git body. Commit refreshed TOML before rendering; configure Git author identity and install `rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros @@ -88,9 +92,9 @@ Local lockless components skip upstream carryover; Release and changelog remain They exclude the TOML defining the local spec path, while retaining other TOML sidecars. They participate in unchanged-render checks and automatic commits. For local components, `--changelog-comment` supplies only the Git message; otherwise -Git opens its own editor. -`--check-only` never creates project commits. Local components skip upstream -carryover and automatic commits. +rendering collects it before output replacement, with commit-only instructions for non-autorelease. +Local components with `--no-git-commit` never prompt. +`--check-only` never creates project commits. Top-level commands in this build: diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 5d50f9212..177dd4104 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -88,9 +88,13 @@ calculation = "manual" Upstream rendering runs host `rpmdev-bumpspec -c` only for non-autorelease specs and automatically commits the component output directory. Missing -`--changelog-comment` opens `$EDITOR` or a local editor for non-autorelease, -or Git's own editor for an autorelease commit. -The changelog editor starts with two blank lines and `#`-commented instructions. +`--changelog-comment` opens `$EDITOR` or a local editor before output replacement, +including for autorelease commits. +Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. +The final Git commit never prompts; its first message line is the summary, and remaining lines are body. +With no upstream changes, the editor starts with two blank lines and `#`-commented instructions. +Otherwise, non-autorelease changes start on line 2; autorelease puts `...` on line 2 +and starts changes on line 3. Each change line is prefixed with `- ` and is editable. Blank lines and lines starting with `#` are removed, with no trailing newline; the cleaned text is reused for the Git message. Explicit flag values remain unchanged. Commit input TOML first, install host `rpmdevtools`, `rpm-build`, and `rpmautospec` as @@ -103,6 +107,7 @@ Local lockless components skip upstream carryover; Release and changelog remain They exclude the TOML defining the local spec path, retain other TOML sidecars, and participate in unchanged-render checks and automatic commits. For local components, `--changelog-comment` supplies only the Git message. +Local components with `--no-git-commit` never prompt. The first automatic commit also includes pending repository changes outside the rendered output directory; later commits contain only their component output. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index ac280c229..85ee728ba 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -60,7 +60,13 @@ Their Release and changelog are never automatically modified, even with --rpmdev They also reject unchanged renders unless --no-change-rebuild is provided, and automatically commit rendered output unless --no-git-commit is provided. For local components, --changelog-comment supplies only the Git commit message; -without it, Git prompts in its own editor. Output must be inside the project repository.` +without it, rendering collects the message in an editor before replacing output. +--no-git-commit skips this prompt. Output must be inside the project repository.` + +const renderMessageHelp = `Non-autorelease specs use rpmdev-bumpspec -c with --changelog-comment, or edited text. +Autorelease skips bumpspec. Missing messages are collected in an editor before +bumping or replacing output; the final Git commit never prompts. +Autorelease with --no-git-commit needs no message and never prompts.` // NewRenderCmd constructs a [cobra.Command] for the "component render" CLI subcommand. func NewRenderCmd() *cobra.Command { @@ -95,8 +101,7 @@ For lockless upstream components, prepare the changelog, then Release, using the upstream spec and last committed render before writing macros, applying overlays, and updating sources. Overlays can override the prepared release and changelog. Compare output; unchanged renders require --no-change-rebuild to record a timestamp. -Non-autorelease specs use rpmdev-bumpspec -c with --changelog-comment, or edited text. -Autorelease skips bumpspec; without --changelog-comment, Git prompts for a message. +` + renderMessageHelp + ` The first commit also includes all pending changes outside the rendered output directory. Later commits are component-only. --no-git-commit skips staging and committing. Output must be inside the project repository; configure Git author identity and diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 8cf7a5a2b..42c9da341 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -840,6 +840,10 @@ func finishLocklessSpec( } } + if err := prepareRenderMessage(env, path, update); err != nil { + return false, err + } + if update.local { return !changed, nil } @@ -907,13 +911,6 @@ func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate return false, fmt.Errorf("lockless upstream rendering requires 'rpmdev-bumpspec'; install 'rpmdevtools':\n%w", err) } - if !update.subjectProvided && update.subject == "" && !env.DryRun() { - update.subject, err = editRenderChangelog(env, path) - if err != nil { - return false, err - } - } - raw := exec.CommandContext(env, sources.RPMDevBumpspecBinary, "-c", update.subject, path) raw.Dir = filepath.Dir(path) diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 069da4c94..08c59ca5f 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -1236,7 +1236,7 @@ func TestFinishLocalLocklessSpec_DoesNotBump(t *testing.T) { return nil } - changed, err := finishLocklessSpec(env.Env, path, &locklessRenderUpdate{local: true}, + changed, err := finishLocklessSpec(env.Env, path, &locklessRenderUpdate{local: true, noGitCommit: true}, testCase.changed, testCase.rebuild) if testCase.wantError { require.ErrorIs(t, err, errNoRenderChanges) @@ -1256,6 +1256,18 @@ func TestFinishLocalLocklessSpec_DoesNotBump(t *testing.T) { } } +func assertCurlRenderCommit(t *testing.T, args []string, message string, provided bool) { + t.Helper() + + wantArgs := []string{"git", "-C", "/project", "commit", "--only", "-m", message} + if provided { + wantArgs = append(wantArgs, "-m", "abc123 Fix upstream") + } + + wantArgs = append(wantArgs, "--", "SPECS/c/curl") + assert.Equal(t, wantArgs, args) +} + func TestPrepareComponentSources_ChangelogAndReleaseBeforeOverlays(t *testing.T) { const releaseTag = "Release: " @@ -1451,10 +1463,8 @@ upstream-commit = "aaaaaaa" } if cmd.Args[3] == "commit" { - assert.Equal(t, []string{ - "git", "-C", "/project", "commit", "--only", "-m", testCase.wantComment, - "-m", "abc123 Fix upstream", "--", "SPECS/c/curl", - }, cmd.Args) + assertCurlRenderCommit(t, cmd.Args, testCase.wantComment, + testCase.explicit || testCase.comment != "") } return "", nil @@ -1896,7 +1906,7 @@ func TestCommitLocklessRender_PathOnlyAndNoEmptyCommits(t *testing.T) { require.Len(t, commands, 3) assert.Equal(t, []string{ "git", "-C", "/project", "commit", "--only", "-m", "updated curl", - "-m", "abc123 Fix", "--", "SPECS/c/curl", + "--", "SPECS/c/curl", }, commands[2]) } else { assert.Len(t, commands, 2) diff --git a/internal/app/azldev/cmds/component/render_message.go b/internal/app/azldev/cmds/component/render_message.go index 954da6014..b616b7fe7 100644 --- a/internal/app/azldev/cmds/component/render_message.go +++ b/internal/app/azldev/cmds/component/render_message.go @@ -4,6 +4,7 @@ package component import ( + "bytes" "errors" "fmt" "log/slog" @@ -14,19 +15,42 @@ import ( "github.com/kballard/go-shellquote" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" ) const renderChangelogTemplate = "\n\n" + - "# Please enter a changelog entry message (one or more lines).\n" + - "# Blank lines and lines starting with # will be ignored.\n" + "# Please enter a changelog entry message. The text will be used\n" + + "# unmodified for the changelog entry (other than the first line\n" + + "# automatically getting a prefix of '- '. This will also be used for\n" + + "# the git commit message.\n" + +const renderNoGitChangelogTemplate = "\n\n" + + "# Please enter a changelog entry message. The text will be used\n" + + "# unmodified for the changelog entry (other than the first line\n" + + "# automatically getting a prefix of '- '.\n" + +const renderAutoreleaseTemplate = "\n\n" + + "# Please enter a changelog entry message. The first line will be used\n" + + "# for the changelog entry and git commit summary, and the following\n" + + "# lines will be used as the git commit message.\n" + + "#\n" + + "# To include more lines in the changelog entry, use this reference:\n" + + "# https://fedora-infra.github.io/rpmautospec-docs/autochangelog.html" + + "#changelog-entries-generated-from-commit-messages\n" + +const renderLocalCommitTemplate = "\n\n" + + "# Please provide a git commit message. This is not used for the\n" + + "# changelog, this is used only for the git commit.\n" func renderFinishConcurrency(env *azldev.Env, prepared []*preparedComponent, checkOnly bool) int { if !checkOnly && !env.DryRun() { for _, prep := range prepared { - if prep.update != nil && !prep.update.local && !prep.update.subjectProvided && prep.update.subject == "" { + if prep.update != nil && (!prep.update.local || !prep.update.noGitCommit) && + !prep.update.subjectProvided && prep.update.subject == "" { return 1 } } @@ -58,34 +82,91 @@ func renderEditor(env *azldev.Env) ([]string, error) { return nil, errors.New("no local editor found; set 'EDITOR' or provide '--changelog-comment'") } -func editRenderChangelog(env *azldev.Env, specPath string) (string, error) { +func prepareRenderMessage(env *azldev.Env, specPath string, update *locklessRenderUpdate) error { + if update.subjectProvided || update.subject != "" || env.DryRun() || (update.local && update.noGitCommit) { + return nil + } + + content, err := fileutils.ReadFile(env.FS(), specPath) + if err != nil { + return fmt.Errorf("reading spec for render message:\n%w", err) + } + + opened, err := spec.OpenSpec(bytes.NewReader(content)) + if err != nil { + return fmt.Errorf("parsing spec for render message:\n%w", err) + } + + release, err := opened.GetLastTag("", "Release") + if err != nil { + return fmt.Errorf("reading Release for render message:\n%w", err) + } + + autorelease := sources.ReleaseUsesAutorelease(release) + if autorelease && update.noGitCommit { + return nil + } + + template := renderChangelogTemplate + + switch { + case autorelease: + template = renderAutoreleaseTemplate + case update.local: + template = renderLocalCommitTemplate + case update.noGitCommit: + template = renderNoGitChangelogTemplate + } + + update.subject, err = editRenderMessage(env, specPath, prefillRenderChanges(template, update.changes, autorelease)) + + return err +} + +func prefillRenderChanges(template, changes string, autorelease bool) string { + if strings.TrimSpace(changes) == "" { + return template + } + + prefix := "\n" + if autorelease { + prefix += "...\n" + } + + changes = strings.TrimSuffix(changes, "\n") + + return prefix + "- " + strings.ReplaceAll(changes, "\n", "\n- ") + + "\n\n" + strings.TrimPrefix(template, "\n\n") +} + +func editRenderMessage(env *azldev.Env, specPath, template string) (string, error) { args, err := renderEditor(env) if err != nil { return "", err } - path, cleanup, err := renderMessageFile(env, renderChangelogTemplate) + path, cleanup, err := renderMessageFile(env, template) if err != nil { return "", err } defer cleanup() - slog.Info("Enter changelog text, save and close the editor; blank lines and lines starting with # are ignored", + slog.Info("Enter render message, save and close the editor; blank lines and lines starting with # are ignored", "spec", specPath) raw := exec.CommandContext(env, args[0], append(args[1:], path)...) if err := runRenderInteractive(env, raw); err != nil { - return "", fmt.Errorf("editing render changelog:\n%w", err) + return "", fmt.Errorf("editing render message:\n%w", err) } content, err := fileutils.ReadFile(env.FS(), path) if err != nil { - return "", fmt.Errorf("reading edited render changelog:\n%w", err) + return "", fmt.Errorf("reading edited render message:\n%w", err) } message := cleanRenderChangelog(string(content)) if message == "" { - return "", errors.New("no changelog text provided; rendering will not be done") + return "", errors.New("no render message provided; rendering will not be done") } return message, nil @@ -144,33 +225,26 @@ func runRenderInteractive(env *azldev.Env, raw *exec.Cmd) error { } func runLocklessRenderCommit(env *azldev.Env, update *locklessRenderUpdate, paths []string) error { - args := []string{"commit", "--only"} - if update.subjectProvided || update.subject != "" { - args = append(args, "-m", update.subject) - if update.changes != "" { - args = append(args, "-m", update.changes) - } - - args = append(args, "--") - args = append(args, paths...) - - _, err := git.RunInDir(env, env, update.repoRoot, args...) - if err != nil { - return fmt.Errorf("running render commit:\n%w", err) - } + if !update.subjectProvided && update.subject == "" { + return errors.New("no render message collected before Git commit") + } - return nil + message := update.subject + if summary, body, multiline := strings.Cut(message, "\n"); multiline && !strings.HasPrefix(body, "\n") { + message = summary + "\n\n" + body } - path, cleanup, err := renderMessageFile(env, "\n\n"+update.changes+"\n") - if err != nil { - return err + args := []string{"commit", "--only", "-m", message} + if update.subjectProvided && update.changes != "" { + args = append(args, "-m", update.changes) } - defer cleanup() - args = append([]string{"-C", update.repoRoot}, args...) - args = append(args, "--template", path, "--") + args = append(args, "--") args = append(args, paths...) - return runRenderInteractive(env, exec.CommandContext(env, "git", args...)) + if _, err := git.RunInDir(env, env, update.repoRoot, args...); err != nil { + return fmt.Errorf("running render commit:\n%w", err) + } + + return nil } diff --git a/internal/app/azldev/cmds/component/render_message_internal_test.go b/internal/app/azldev/cmds/component/render_message_internal_test.go index 7ac2a0532..96518c5ff 100644 --- a/internal/app/azldev/cmds/component/render_message_internal_test.go +++ b/internal/app/azldev/cmds/component/render_message_internal_test.go @@ -7,7 +7,6 @@ import ( "errors" "os" "os/exec" - "slices" "strings" "testing" @@ -60,39 +59,45 @@ func TestRenderEditorSelection(t *testing.T) { } } -func TestBumpLocklessSpecMessageAndAutorelease(t *testing.T) { +func TestFinishLocklessSpecMessages(t *testing.T) { for _, testCase := range []struct { - name, release, message, wantMessage string - provided, autorelease bool + name, release, message, template string + provided, local, noGit bool }{ - { - name: "edited text", release: "1", message: "Fix feature\n\nKeep details.\n", - wantMessage: "Fix feature\nKeep details.", - }, + {name: "upstream static commit", release: "1", template: renderChangelogTemplate}, + {name: "upstream static no commit", release: "1", noGit: true, template: renderNoGitChangelogTemplate}, + {name: "upstream autorelease commit", release: "%autorelease -b 9", template: renderAutoreleaseTemplate}, + {name: "upstream autorelease no commit", release: "%autorelease", noGit: true}, + {name: "local static commit", release: "9", local: true, template: renderLocalCommitTemplate}, + {name: "local static no commit", release: "9", local: true, noGit: true}, + {name: "local autorelease commit", release: "%autorelease", local: true, template: renderAutoreleaseTemplate}, + {name: "local autorelease no commit", release: "%autorelease", local: true, noGit: true}, {name: "provided text", release: "1", message: "Explicit entry", provided: true}, {name: "provided multiline text", release: "1", message: "# Explicit\n\nKeep details.\n", provided: true}, {name: "explicit empty value", release: "1", provided: true}, - {name: "autorelease without message", release: "%autorelease -b 9", autorelease: true}, - {name: "autorelease with message", release: "%autorelease", message: "Git only", provided: true, autorelease: true}, + {name: "provided autorelease", release: "%autorelease", message: "Git only", provided: true}, + {name: "provided local", release: "9", local: true, message: "Git only", provided: true}, } { t.Run(testCase.name, func(t *testing.T) { env := testutils.NewTestEnv(t) setRenderEditor(t, env, "edit-tool --wait") env.CmdFactory.RegisterCommandInSearchPath("edit-tool") - if !testCase.autorelease { + wantBump := !testCase.local && !strings.Contains(testCase.release, "%autorelease") + if wantBump { env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") } const specPath = "/working/curl.spec" - require.NoError(t, fileutils.WriteFile(env.TestFS, specPath, - []byte(renderTestSpec("1", testCase.release, "")), fileperms.PublicFile)) + + original := renderTestSpec("1", testCase.release, "* Preserve changelog\n- Existing entry\n") + require.NoError(t, fileutils.WriteFile(env.TestFS, specPath, []byte(original), fileperms.PublicFile)) editCalls, bumpCalls := 0, 0 wantMessage := testCase.message - if !testCase.provided && !testCase.autorelease { - wantMessage = testCase.wantMessage + if testCase.template != "" { + wantMessage = "Fix feature\nKeep details." } var messagePath string @@ -108,16 +113,18 @@ func TestBumpLocklessSpecMessageAndAutorelease(t *testing.T) { content, err := fileutils.ReadFile(env.TestFS, messagePath) require.NoError(t, err) - assert.True(t, strings.HasPrefix(string(content), "\n\n#")) - assert.Contains(t, string(content), "enter a changelog entry message") - assert.Contains(t, string(content), "Blank lines and lines starting with # will be ignored") + assert.Equal(t, testCase.template, string(content)) return fileutils.WriteFile(env.TestFS, messagePath, - []byte(testCase.message+string(content)), fileperms.PrivateFile) + []byte("Fix feature\n\nKeep details.\n"+string(content)), fileperms.PrivateFile) case "rpmdev-bumpspec": bumpCalls++ assert.Equal(t, []string{"rpmdev-bumpspec", "-c", wantMessage, specPath}, cmd.Args) + + if testCase.template != "" { + assert.Equal(t, 1, editCalls, "collect the message before bumpspec") + } default: t.Fatalf("unexpected command: %v", cmd.Args) } @@ -125,31 +132,36 @@ func TestBumpLocklessSpecMessageAndAutorelease(t *testing.T) { return nil } - update := &locklessRenderUpdate{subjectProvided: testCase.provided} + update := &locklessRenderUpdate{ + subjectProvided: testCase.provided, local: testCase.local, noGitCommit: testCase.noGit, + } if testCase.provided { update.subject = testCase.message } - _, err := bumpLocklessSpec(env.Env, specPath, update) + _, err := finishLocklessSpec(env.Env, specPath, update, true, false) require.NoError(t, err) + assert.Equal(t, wantMessage, update.subject) - if testCase.autorelease { - assert.Zero(t, editCalls) - assert.Zero(t, bumpCalls) - } else { + if wantBump { assert.Equal(t, 1, bumpCalls) - assert.Equal(t, wantMessage, update.subject) + } else { + assert.Zero(t, bumpCalls) + } - if testCase.provided { - assert.Zero(t, editCalls) - } else { - assert.Equal(t, 1, editCalls) + if testCase.template == "" { + assert.Zero(t, editCalls) + } else { + assert.Equal(t, 1, editCalls) - exists, err := fileutils.Exists(env.TestFS, messagePath) - require.NoError(t, err) - assert.False(t, exists) - } + exists, err := fileutils.Exists(env.TestFS, messagePath) + require.NoError(t, err) + assert.False(t, exists) } + + content, err := fileutils.ReadFile(env.TestFS, specPath) + require.NoError(t, err) + assert.Equal(t, original, string(content)) }) } } @@ -188,66 +200,169 @@ func TestCleanRenderChangelog(t *testing.T) { } } -func TestEditRenderChangelogRejectsEmptyAndFailedEditor(t *testing.T) { +func TestFinishLocklessSpecRejectsEmptyAndFailedEditor(t *testing.T) { editorErr := errors.New("editor failed") - for _, fail := range []bool{false, true} { - t.Run(map[bool]string{false: "unchanged template", true: "failure"}[fail], func(t *testing.T) { + for _, testCase := range []struct { + name, response string + fail bool + }{ + {name: "blank"}, + {name: "only blank lines", response: "\n \t\n"}, + {name: "only comments and blank lines", response: "\n# Instructions\n#\n \t\n"}, + {name: "failure", fail: true}, + } { + t.Run(testCase.name, func(t *testing.T) { env := testutils.NewTestEnv(t) setRenderEditor(t, env, "edit-tool") env.CmdFactory.RegisterCommandInSearchPath("edit-tool") + const specPath = "/working/curl.spec" + + original := renderTestSpec("1", "9", "") + require.NoError(t, fileutils.WriteFile(env.TestFS, specPath, []byte(original), fileperms.PublicFile)) + var messagePath string env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + require.Equal(t, "edit-tool", cmd.Args[0], "must fail before bumpspec or Git") messagePath = cmd.Args[1] - if fail { + if testCase.fail { return editorErr } - return nil + return fileutils.WriteFile(env.TestFS, messagePath, []byte(testCase.response), fileperms.PrivateFile) } - _, err := editRenderChangelog(env.Env, "/working/curl.spec") - if fail { + _, err := finishLocklessSpec(env.Env, specPath, + &locklessRenderUpdate{changes: "abc123 Upstream fix"}, true, false) + if testCase.fail { require.ErrorIs(t, err, editorErr) } else { - require.ErrorContains(t, err, "no changelog text") + require.ErrorContains(t, err, "no render message provided") } exists, err := fileutils.Exists(env.TestFS, messagePath) require.NoError(t, err) assert.False(t, exists) + + content, err := fileutils.ReadFile(env.TestFS, specPath) + require.NoError(t, err) + assert.Equal(t, original, string(content)) }) } } -func TestLocklessRenderCommitPromptsWithoutMessage(t *testing.T) { +func TestLocklessRenderCommitRejectsMissingMessage(t *testing.T) { env := testutils.NewTestEnv(t) - var templatePath string - env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { - assert.Equal(t, []string{"git", "-C", "/project", "commit", "--only"}, cmd.Args[:5]) - assert.NotContains(t, cmd.Args, "-m") - assert.Contains(t, cmd.Args, "SPECS/c/curl") - assert.Contains(t, cmd.Args, ":(top,literal)inputs.toml") - assert.Same(t, os.Stdin, cmd.Stdin) - assert.Same(t, os.Stdout, cmd.Stdout) - assert.Same(t, os.Stderr, cmd.Stderr) - templatePath = cmd.Args[slices.Index(cmd.Args, "--template")+1] - content, err := fileutils.ReadFile(env.TestFS, templatePath) - require.NoError(t, err) - assert.Contains(t, string(content), "abc123 Upstream fix") + t.Fatalf("must not open Git's editor: %v", cmd.Args) return nil } - require.NoError(t, runLocklessRenderCommit(env.Env, + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("must not run Git without a message: %v", cmd.Args) + + return "", nil + } + require.ErrorContains(t, runLocklessRenderCommit(env.Env, &locklessRenderUpdate{repoRoot: "/project", changes: "abc123 Upstream fix"}, - []string{"SPECS/c/curl", ":(top,literal)inputs.toml"})) - require.NotEmpty(t, templatePath) - exists, err := fileutils.Exists(env.TestFS, templatePath) - require.NoError(t, err) - assert.False(t, exists) + []string{"SPECS/c/curl", ":(top,literal)inputs.toml"}), "no render message collected") +} + +func TestLocklessRenderCommitReusesMessage(t *testing.T) { + for _, message := range []string{"Summary\nBody", "Summary\n\nBody"} { + t.Run(message, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("must not open Git's editor: %v", cmd.Args) + + return nil + } + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{ + "git", "-C", "/project", "commit", "--only", "-m", "Summary\n\nBody", + "--", "SPECS/c/curl", + }, cmd.Args) + + return "", nil + } + require.NoError(t, runLocklessRenderCommit(env.Env, &locklessRenderUpdate{ + repoRoot: "/project", subject: message, changes: "abc123 Upstream fix", + }, []string{"SPECS/c/curl"})) + }) + } +} + +func TestRenderMessagePrefillsUpstreamChanges(t *testing.T) { + const autorelease = "%autorelease" + + for _, testCase := range []struct { + name, release, changes, prefix, message string + }{ + { + name: "static", release: "1", changes: "abc123 First fix\ndef456 Second fix", + prefix: "\n- abc123 First fix\n- def456 Second fix\n\n", + message: "Summary\n- abc123 First fix\n- def456 Second fix", + }, + { + name: "autorelease", release: autorelease, changes: "abc123 First fix\ndef456 Second fix\n", + prefix: "\n...\n- abc123 First fix\n- def456 Second fix\n\n", + message: "Summary\n...\n- abc123 First fix\n- def456 Second fix", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setRenderEditor(t, env, "edit-tool") + env.CmdFactory.RegisterCommandInSearchPath("edit-tool") + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + const specPath = "/working/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, specPath, + []byte(renderTestSpec("1", testCase.release, "")), fileperms.PublicFile)) + + editorCalls := 0 + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + if cmd.Args[0] == "rpmdev-bumpspec" { + assert.Equal(t, []string{"rpmdev-bumpspec", "-c", testCase.message, specPath}, cmd.Args) + + return nil + } + + require.Equal(t, "edit-tool", cmd.Args[0]) + + editorCalls++ + content, err := fileutils.ReadFile(env.TestFS, cmd.Args[1]) + require.NoError(t, err) + + template := renderChangelogTemplate + if testCase.release == autorelease { + template = renderAutoreleaseTemplate + } + + assert.Equal(t, testCase.prefix+strings.TrimPrefix(template, "\n\n"), string(content)) + + return fileutils.WriteFile(env.TestFS, cmd.Args[1], []byte("Summary"+string(content)), + fileperms.PrivateFile) + } + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + summary, body, _ := strings.Cut(testCase.message, "\n") + assert.Equal(t, []string{ + "git", "-C", "/project", "commit", "--only", "-m", summary + "\n\n" + body, + "--", "SPECS/c/curl", + }, cmd.Args) + + return "", nil + } + + update := &locklessRenderUpdate{repoRoot: "/project", changes: testCase.changes} + _, err := finishLocklessSpec(env.Env, specPath, update, true, false) + require.NoError(t, err) + assert.Equal(t, testCase.message, update.subject) + require.NoError(t, runLocklessRenderCommit(env.Env, update, []string{"SPECS/c/curl"})) + assert.Equal(t, 1, editorCalls) + }) + } } From 39bcf22692b0e3fab7e99d9d844cf8c729f5854b Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 6 Nov 2026 19:20:29 +0000 Subject: [PATCH 40/70] fix(component): restrict rebuild timestamps to autorelease Write no-change rebuild timestamps only for autorelease components, whether upstream or local. Unchanged upstream static specs proceed to their normal release and changelog bump without a marker. Reject unchanged local non-autorelease components, even with the rebuild flag, explaining that their release and changelog require manual updates. Classify the final post-overlay Release consistently for rebuilds and message prompts. Update focused tests, user documentation, generated CLI help, and emitted agent guidance. Unit tests, complete quality checks, and native rebuild smoke tests passed. Docker scenarios and snapshot updates remain blocked by host Docker-provider initialization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 18 ++++--- .../reference/cli/azldev_component_render.md | 10 ++-- .../app/azldev/agentskill/agentskill_test.go | 4 ++ .../azldev/agentskill/content/azldev.md.tmpl | 7 ++- .../content/refresh-upstream-commit.md.tmpl | 7 ++- .../content/withoutlockfile/azldev.md.tmpl | 7 ++- internal/app/azldev/cmds/component/render.go | 10 ++-- .../cmds/component/render_internal_test.go | 15 +++--- .../azldev/cmds/component/render_lockless.go | 50 +++++++++++++++++-- .../render_lockless_internal_test.go | 26 ++++++---- .../azldev/cmds/component/render_message.go | 18 +------ 11 files changed, 112 insertions(+), 60 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 7b459adba..5a7d9611a 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -263,19 +263,23 @@ An error reading the current pin is fatal. It then compares the prepared tree with existing output, ignoring the root `.git` and any top-level `.no-change-rebuild` on both sides. `--check-only` stops here, before the final release bump. -If normal rendering detects no changes, it returns an error indicating there -has been no change since the last render, without replacing the existing output -or writing a failure marker. To explicitly rebuild unchanged content, +If normal rendering detects no changes, it returns an error without replacing +the existing output or writing a failure marker. Unchanged local non-autorelease +components must manually update their release and changelog; they fail even with +`--no-change-rebuild`. To explicitly rebuild unchanged upstream content or a local +autorelease component, pass `--no-change-rebuild`, a boolean switch taking no argument: ```bash azldev comp render -p curl --no-change-rebuild ``` -For an unchanged render with this flag, the tool runs host `date -Is` and writes -its exact output to `.no-change-rebuild` at the working dist-git root before -bumping upstream specs. Local specs are not bumped. The marker is then published -and committed with the rendered files. +For an unchanged autorelease component (upstream or local) with this flag, the +tool runs host `date -Is` and writes its exact output to `.no-change-rebuild` at +the working dist-git root. The marker is then published and committed with the +rendered files. Unchanged upstream non-autorelease components instead proceed +to the normal release/changelog bump without writing a timestamp marker. +Local specs are never automatically bumped. The flag does not create a marker when comparison detects real changes. The root marker is ignored in both the working upstream checkout and existing output, even if its contents differ. Nested files with the same name are compared. diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 17471aec1..176216141 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -29,7 +29,8 @@ use lock files. For lockless upstream components, prepare the changelog, then Release, using the upstream spec and last committed render before writing macros, applying overlays, and updating sources. Overlays can override the prepared release and changelog. -Compare output; unchanged renders require --no-change-rebuild to record a timestamp. +Compare output; --no-change-rebuild permits unchanged upstream renders and records +a timestamp only for autorelease components (upstream or local). Non-autorelease specs use rpmdev-bumpspec -c with --changelog-comment, or edited text. Autorelease skips bumpspec. Missing messages are collected in an editor before bumping or replacing output; the final Git commit never prompts. @@ -48,8 +49,9 @@ and reused for the macros file. Locked components keep their existing behavior. Local lockless components copy local files except the TOML defining the spec path. Their Release and changelog are never automatically modified, even with --rpmdev-bumpspec. -They also reject unchanged renders unless --no-change-rebuild is provided, and -automatically commit rendered output unless --no-git-commit is provided. +Unchanged local non-autorelease components must manually update Release and changelog. +Local autorelease components can use --no-change-rebuild to record a timestamp. +Rendered output is automatically committed unless --no-git-commit is provided. For local components, --changelog-comment supplies only the Git commit message; without it, rendering collects the message in an editor before replacing output. --no-git-commit skips this prompt. Output must be inside the project repository. @@ -99,7 +101,7 @@ azldev component render [flags] --fail-on-error exit with error if any component fails to render (useful for CI) -f, --force allow overwriting existing rendered component directories -h, --help help for render - --no-change-rebuild allow unchanged lockless renders by recording 'date -Is' output in '.no-change-rebuild' + --no-change-rebuild allow unchanged upstream renders and local autorelease renders; record a timestamp only for autorelease --no-git-commit leave lockless rendered output uncommitted without staging it -o, --output-dir string output directory for rendered specs (overrides rendered-specs-dir from config) --rpmdev-bumpspec Use rpmdev-bumpspec instead of the legacy static release calculation diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 11fc70e2f..61b1909f5 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -212,6 +212,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "TOML defining the local spec path") assert.Contains(t, doc, "participate in unchanged-render checks and automatic commits") assert.Contains(t, doc, "host `date -Is` output") + assert.Contains(t, doc, "Upstream non-autorelease rebuilds proceed without a timestamp marker") + assert.Contains(t, doc, "must manually update their release and changelog") assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") @@ -264,6 +266,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") assert.Contains(t, doc, "Unchanged lockless renders require `--no-change-rebuild`") assert.Contains(t, doc, "host `date -Is` output") + assert.Contains(t, doc, "Upstream non-autorelease rebuilds proceed without a timestamp marker") + assert.Contains(t, doc, "must manually update their release and changelog") assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index dc2e9ad37..c77283b16 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -59,8 +59,11 @@ release adjustment. Lockless rendering preserves the existing spaces and tabs after `Release:` when editing its value, and compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless renders require `--no-change-rebuild`; otherwise rendering errors -without replacing existing output. The flag records host `date -Is` output in -`.no-change-rebuild` before bumping, only when no changes were detected. Comparison +without replacing existing output. For unchanged autorelease components (upstream or local), +the flag records host `date -Is` output in `.no-change-rebuild`. +Upstream non-autorelease rebuilds proceed without a timestamp marker. +Unchanged local non-autorelease components must manually update their release and changelog, +even with `--no-change-rebuild`. Comparison ignores this root marker in both existing output and the working upstream checkout. Lockless failures never write `RENDER_FAILED`. Before replacement, output is untouched; failures after replacement starts clear the component directory and diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 4fba9d47a..51db686d2 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -100,8 +100,11 @@ Overlays can override the prepared release and changelog. Carryover is not repeated after overlays, even if they change the version. Rendering compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless renders require `--no-change-rebuild`; otherwise rendering errors -without replacing existing output. The flag records host `date -Is` output in -`.no-change-rebuild` before bumping, only when no changes were detected. Comparison +without replacing existing output. For unchanged autorelease components (upstream or local), +the flag records host `date -Is` output in `.no-change-rebuild`. +Upstream non-autorelease rebuilds proceed without a timestamp marker. +Unchanged local non-autorelease components must manually update their release and changelog, +even with `--no-change-rebuild`. Comparison ignores this root marker in both existing output and the working upstream checkout. Lockless failures never write `RENDER_FAILED`. Before replacement, output is untouched; failures after replacement starts clear the component directory and diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index dfccae214..685b005b1 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -54,8 +54,11 @@ Non-autorelease rendering then runs host `rpmdev-bumpspec`. Release edits preserve the existing spaces and tabs after `Release:`. Rendering compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless renders require `--no-change-rebuild`; otherwise rendering errors -without replacing existing output. The flag records host `date -Is` output in -`.no-change-rebuild` before bumping, only when no changes were detected. Comparison +without replacing existing output. For unchanged autorelease components (upstream or local), +the flag records host `date -Is` output in `.no-change-rebuild`. +Upstream non-autorelease rebuilds proceed without a timestamp marker. +Unchanged local non-autorelease components must manually update their release and changelog, +even with `--no-change-rebuild`. Comparison ignores this root marker in both existing output and the working upstream checkout. Lockless failures never write `RENDER_FAILED`. Before replacement, output is untouched; failures after replacement starts clear the component directory and diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 85ee728ba..4bf94de78 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -57,8 +57,9 @@ rendering; Git-ignored files are excluded. --allow-uncommitted bypasses this che const renderLocalHelp = `Local lockless components copy local files except the TOML defining the spec path. Their Release and changelog are never automatically modified, even with --rpmdev-bumpspec. -They also reject unchanged renders unless --no-change-rebuild is provided, and -automatically commit rendered output unless --no-git-commit is provided. +Unchanged local non-autorelease components must manually update Release and changelog. +Local autorelease components can use --no-change-rebuild to record a timestamp. +Rendered output is automatically committed unless --no-git-commit is provided. For local components, --changelog-comment supplies only the Git commit message; without it, rendering collects the message in an editor before replacing output. --no-git-commit skips this prompt. Output must be inside the project repository.` @@ -100,7 +101,8 @@ use lock files. For lockless upstream components, prepare the changelog, then Release, using the upstream spec and last committed render before writing macros, applying overlays, and updating sources. Overlays can override the prepared release and changelog. -Compare output; unchanged renders require --no-change-rebuild to record a timestamp. +Compare output; --no-change-rebuild permits unchanged upstream renders and records +a timestamp only for autorelease components (upstream or local). ` + renderMessageHelp + ` The first commit also includes all pending changes outside the rendered output directory. Later commits are component-only. --no-git-commit skips staging and committing. @@ -172,7 +174,7 @@ func addRenderOptions(cmd *cobra.Command, options *RenderOptions) { "and 1 when any component would drift. With -a + --clean-stale, also fails "+ "on orphan rendered-spec directories. Intended for CI gates.") cmd.Flags().BoolVar(&options.NoChangeRebuild, "no-change-rebuild", false, - "allow unchanged lockless renders by recording 'date -Is' output in '.no-change-rebuild'") + "allow unchanged upstream renders and local autorelease renders; record a timestamp only for autorelease") cmd.Flags().StringVar(&options.ChangelogComment, "changelog-comment", "", "Git commit message for lockless components and verbatim rpmdev-bumpspec -c comment "+ "for upstream non-autorelease specs (if omitted, prompt in an editor)") diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index b5c107dba..2af724eed 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -804,7 +804,7 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { dated := false wantFinish := !testCase.checkOnly && !testCase.compareFail && !testCase.wantNoChanges wantBump := wantFinish && !testCase.autorelease - wantMarker := wantFinish && !testCase.drift + wantMarker := wantFinish && !testCase.drift && testCase.autorelease env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { assert.True(t, comparisonFS.compared, "comparison must precede the bump") @@ -813,11 +813,9 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { assert.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) - if wantMarker { - marker, err := fileutils.ReadFile(baseFS, filepath.Join(filepath.Dir(stagedSpec), noChangeRebuildMarker)) - require.NoError(t, err) - assert.Equal(t, timestamp, string(marker), "timestamp must be written before bumping") - } + markerExists, err := fileutils.Exists(baseFS, filepath.Join(filepath.Dir(stagedSpec), noChangeRebuildMarker)) + require.NoError(t, err) + assert.False(t, markerExists, "static release bumps must not create a timestamp") bumped := strings.Replace(original, "Release: 9\n", "Release: 10\n", 1) @@ -836,7 +834,10 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { } if !testCase.checkOnly { env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") - env.CmdFactory.RegisterCommandInSearchPath("date") + + if testCase.autorelease { + env.CmdFactory.RegisterCommandInSearchPath("date") + } } prep := &preparedComponent{ diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 42c9da341..128fef366 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -35,6 +35,10 @@ const noChangeRebuildMarker = ".no-change-rebuild" var errNoRenderChanges = errors.New("no change since the last render; use '--no-change-rebuild' to rebuild anyway") +var errLocalManualRebuild = errors.New( + "local non-autorelease components must manually update their release and changelog before rendering again", +) + type locklessRenderUpdate struct { repoRoot string path string @@ -831,11 +835,7 @@ func finishLocklessSpec( env *azldev.Env, path string, update *locklessRenderUpdate, changed, noChangeRebuild bool, ) (bool, error) { if !changed { - if !noChangeRebuild { - return false, errNoRenderChanges - } - - if err := writeNoChangeRebuild(env, filepath.Dir(path)); err != nil { + if err := prepareLocklessRebuild(env, path, update.local, noChangeRebuild); err != nil { return false, err } } @@ -853,6 +853,46 @@ func finishLocklessSpec( return !changed || bumped, err } +func prepareLocklessRebuild(env *azldev.Env, path string, local, noChangeRebuild bool) error { + autorelease, err := locklessSpecUsesAutorelease(env, path) + if err != nil { + return err + } + + if local && !autorelease { + return errLocalManualRebuild + } + + if !noChangeRebuild { + return errNoRenderChanges + } + + if !autorelease { + return nil + } + + return writeNoChangeRebuild(env, filepath.Dir(path)) +} + +func locklessSpecUsesAutorelease(env *azldev.Env, path string) (bool, error) { + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return false, fmt.Errorf("reading spec for lockless render:\n%w", err) + } + + opened, err := spec.OpenSpec(bytes.NewReader(content)) + if err != nil { + return false, fmt.Errorf("parsing spec for lockless render:\n%w", err) + } + + release, err := opened.GetLastTag("", "Release") + if err != nil { + return false, fmt.Errorf("reading Release for lockless render:\n%w", err) + } + + return sources.ReleaseUsesAutorelease(release), nil +} + func writeNoChangeRebuild(env *azldev.Env, dir string) error { if env.DryRun() { return nil diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 08c59ca5f..1d2f5bb6d 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -1210,20 +1210,24 @@ func TestPrepareComponentSources_LocalLocklessKeepsSpecAndSetsCommitState(t *tes func TestFinishLocalLocklessSpec_DoesNotBump(t *testing.T) { for _, testCase := range []struct { - name string + name, release string changed, rebuild bool - wantError bool + wantError error + wantMarker bool }{ - {name: "changed", changed: true}, - {name: "unchanged", wantError: true}, - {name: "explicit rebuild", rebuild: true}, + {name: "changed static", release: "7%{?dist}", changed: true}, + {name: "unchanged static", release: "7%{?dist}", wantError: errLocalManualRebuild}, + {name: "static rebuild refused", release: "7%{?dist}", rebuild: true, wantError: errLocalManualRebuild}, + {name: "unchanged autorelease", release: "%autorelease -b 7", wantError: errNoRenderChanges}, + {name: "autorelease rebuild", release: "%autorelease -b 7", rebuild: true, wantMarker: true}, + {name: "changed autorelease", release: "%autorelease -b 7", changed: true, rebuild: true}, } { t.Run(testCase.name, func(t *testing.T) { env := testutils.NewTestEnv(t) env.CmdFactory.RegisterCommandInSearchPath("date") path := "/working/curl.spec" - content := renderTestSpec("1", "7%{?dist}", "* Keep local changelog\n") + content := renderTestSpec("1", testCase.release, "* Keep local changelog\n") require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(content), fileperms.PublicFile)) env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { assert.Equal(t, []string{"date", "-Is"}, cmd.Args) @@ -1238,11 +1242,11 @@ func TestFinishLocalLocklessSpec_DoesNotBump(t *testing.T) { changed, err := finishLocklessSpec(env.Env, path, &locklessRenderUpdate{local: true, noGitCommit: true}, testCase.changed, testCase.rebuild) - if testCase.wantError { - require.ErrorIs(t, err, errNoRenderChanges) + if testCase.wantError != nil { + require.ErrorIs(t, err, testCase.wantError) } else { require.NoError(t, err) - assert.Equal(t, testCase.rebuild, changed) + assert.Equal(t, testCase.wantMarker, changed) } got, err := fileutils.ReadFile(env.TestFS, path) @@ -1251,7 +1255,7 @@ func TestFinishLocalLocklessSpec_DoesNotBump(t *testing.T) { exists, err := fileutils.Exists(env.TestFS, "/working/.no-change-rebuild") require.NoError(t, err) - assert.Equal(t, testCase.rebuild, exists) + assert.Equal(t, testCase.wantMarker, exists) }) } } @@ -1612,7 +1616,7 @@ func TestFinishLocklessSpec_TimestampFailures(t *testing.T) { const path = "/working/curl.spec" - original := renderTestSpec("1", "9", "") + original := renderTestSpec("1", "%autorelease -b 9", "") require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(original), fileperms.PublicFile)) if testCase.readOnly { diff --git a/internal/app/azldev/cmds/component/render_message.go b/internal/app/azldev/cmds/component/render_message.go index b616b7fe7..cd7dd1973 100644 --- a/internal/app/azldev/cmds/component/render_message.go +++ b/internal/app/azldev/cmds/component/render_message.go @@ -4,7 +4,6 @@ package component import ( - "bytes" "errors" "fmt" "log/slog" @@ -15,8 +14,6 @@ import ( "github.com/kballard/go-shellquote" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" - "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" - "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" @@ -87,22 +84,11 @@ func prepareRenderMessage(env *azldev.Env, specPath string, update *locklessRend return nil } - content, err := fileutils.ReadFile(env.FS(), specPath) + autorelease, err := locklessSpecUsesAutorelease(env, specPath) if err != nil { - return fmt.Errorf("reading spec for render message:\n%w", err) + return err } - opened, err := spec.OpenSpec(bytes.NewReader(content)) - if err != nil { - return fmt.Errorf("parsing spec for render message:\n%w", err) - } - - release, err := opened.GetLastTag("", "Release") - if err != nil { - return fmt.Errorf("reading Release for render message:\n%w", err) - } - - autorelease := sources.ReleaseUsesAutorelease(release) if autorelease && update.noGitCommit { return nil } From a51a589fd373d1b48b23b2e8c6567e959890cdd3 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 6 Nov 2026 19:29:25 +0000 Subject: [PATCH 41/70] fix(component): reject render prompts without interactive terminals Before opening a required changelog or commit editor, require both stdin and stdout to be TTYs. Fail non-interactive rendering with instructions to use --changelog-comment and/or --no-git-commit. Keep supplied messages and cases that need no message non-interactive. Upstream static specs still require a changelog message when Git commits are disabled. Preserve the existing editor selection and message cleanup. Update user documentation, generated help, emitted guidance, and tests. Unit tests, complete quality checks, and native TTY/non-TTY smoke tests passed. Docker scenarios and snapshot updates remain blocked by host Docker-provider initialization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 5 + .../reference/cli/azldev_component_render.md | 1 + .../app/azldev/agentskill/agentskill_test.go | 1 + .../azldev/agentskill/content/azldev.md.tmpl | 3 + .../agentskill/content/comp-toml.md.tmpl | 2 + .../content/refresh-upstream-commit.md.tmpl | 3 + .../content/withoutlockfile/azldev.md.tmpl | 3 + .../content/withoutlockfile/comp-toml.md.tmpl | 2 + internal/app/azldev/cmds/component/render.go | 3 +- .../azldev/cmds/component/render_message.go | 8 ++ .../component/render_message_internal_test.go | 96 +++++++++++++++++++ 11 files changed, 126 insertions(+), 1 deletion(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 5a7d9611a..aeddd0c7f 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -318,6 +318,11 @@ Each case has its own instructional comments. The final Git commit always reuses the supplied or already collected message; it never opens another editor. `--check-only` and dry runs never open editors. For unattended rendering, provide `--changelog-comment`. +Any required editor prompt first checks that both stdin and stdout are TTYs. +Otherwise rendering fails before opening an editor or replacing output: +non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. +For upstream non-autorelease components, `--no-git-commit` alone is insufficient: +the changelog bump still requires `--changelog-comment`. Prefilled upstream changes are used only as retained in the edited message; the original changes are not appended again to its changelog entry or Git body. diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 176216141..d77f310bb 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -35,6 +35,7 @@ Non-autorelease specs use rpmdev-bumpspec -c with --changelog-comment, or edited Autorelease skips bumpspec. Missing messages are collected in an editor before bumping or replacing output; the final Git commit never prompts. Autorelease with --no-git-commit needs no message and never prompts. +Message prompts require both stdin and stdout to be interactive TTYs. The first commit also includes all pending changes outside the rendered output directory. Later commits are component-only. --no-git-commit skips staging and committing. Output must be inside the project repository; configure Git author identity and diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 61b1909f5..1938002c2 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -173,6 +173,7 @@ func TestComponentTomlSkillDocumentsPerComponentRenderModes(t *testing.T) { assert.Contains(t, doc, "TOML defining the local spec path") assert.Contains(t, doc, "participate in unchanged-render checks and automatic commits") assert.Contains(t, doc, "The final Git commit never prompts") + assert.Contains(t, doc, "both stdin and stdout to be TTYs") assert.Contains(t, doc, "Autorelease with `--no-git-commit` never prompts") assert.Contains(t, doc, "Blank/comment-only edited messages fail rendering") assert.Contains(t, doc, "Each change line is prefixed with `- ` and is editable") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index c77283b16..75d124fad 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -88,6 +88,9 @@ Autorelease skips `rpmdev-bumpspec`; missing messages are collected before outpu Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. The final Git commit never prompts; its first message line is the summary, and remaining lines are body. Provide `--changelog-comment` for unattended rendering. +Required message prompts require both stdin and stdout to be TTYs; otherwise rendering fails. +Non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. +Upstream non-autorelease still needs `--changelog-comment` even with `--no-git-commit`. `--no-git-commit` skips automatic staging and committing, but still renders and bumps non-autorelease specs. Edited upstream changes are not appended again; explicit flag messages retain the upstream Git body. diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index db72264d4..ec6e5fb2d 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -98,6 +98,8 @@ the component output. Missing `--changelog-comment` opens `$EDITOR` or a local editor before output replacement, including for autorelease commits. Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. The final Git commit never prompts; its first message line is the summary, and remaining lines are body. +Required message prompts require both stdin and stdout to be TTYs; otherwise rendering fails. +Non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. With no upstream changes, the editor starts with two blank lines and `#`-commented instructions. Otherwise, non-autorelease changes start on line 2; autorelease puts `...` on line 2 and starts changes on line 3. Each change line is prefixed with `- ` and is editable. diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 51db686d2..158b8c5cd 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -184,6 +184,9 @@ Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited mess The final Git commit never prompts; its first message line is the summary, and remaining lines are body. Edited upstream changes are not appended again; explicit flag messages retain the upstream Git body. Provide `--changelog-comment` for unattended rendering. +Required message prompts require both stdin and stdout to be TTYs; otherwise rendering fails. +Non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. +Upstream non-autorelease still needs `--changelog-comment` even with `--no-git-commit`. `--no-git-commit` skips automatic staging and committing, but still renders and bumps non-autorelease specs. It never creates empty commits. Files outside rendered output may remain staged diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 685b005b1..226bbdbc0 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -81,6 +81,9 @@ Autorelease skips `rpmdev-bumpspec`; missing messages are collected before outpu Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. The final Git commit never prompts; its first message line is the summary, and remaining lines are body. Provide `--changelog-comment` for unattended rendering. +Required message prompts require both stdin and stdout to be TTYs; otherwise rendering fails. +Non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. +Upstream non-autorelease still needs `--changelog-comment` even with `--no-git-commit`. `--no-git-commit` skips automatic staging and committing, but still renders and bumps non-autorelease specs. Edited upstream changes are not appended again; explicit flag messages retain the upstream Git body. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 177dd4104..fcc91639e 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -92,6 +92,8 @@ and automatically commits the component output directory. Missing including for autorelease commits. Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. The final Git commit never prompts; its first message line is the summary, and remaining lines are body. +Required message prompts require both stdin and stdout to be TTYs; otherwise rendering fails. +Non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. With no upstream changes, the editor starts with two blank lines and `#`-commented instructions. Otherwise, non-autorelease changes start on line 2; autorelease puts `...` on line 2 and starts changes on line 3. Each change line is prefixed with `- ` and is editable. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 4bf94de78..d2c143642 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -67,7 +67,8 @@ without it, rendering collects the message in an editor before replacing output. const renderMessageHelp = `Non-autorelease specs use rpmdev-bumpspec -c with --changelog-comment, or edited text. Autorelease skips bumpspec. Missing messages are collected in an editor before bumping or replacing output; the final Git commit never prompts. -Autorelease with --no-git-commit needs no message and never prompts.` +Autorelease with --no-git-commit needs no message and never prompts. +Message prompts require both stdin and stdout to be interactive TTYs.` // NewRenderCmd constructs a [cobra.Command] for the "component render" CLI subcommand. func NewRenderCmd() *cobra.Command { diff --git a/internal/app/azldev/cmds/component/render_message.go b/internal/app/azldev/cmds/component/render_message.go index cd7dd1973..f52297c43 100644 --- a/internal/app/azldev/cmds/component/render_message.go +++ b/internal/app/azldev/cmds/component/render_message.go @@ -13,12 +13,16 @@ import ( "strings" "github.com/kballard/go-shellquote" + "github.com/mattn/go-isatty" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" ) +//nolint:gochecknoglobals // Replaced by isolated render tests without changing real standard streams. +var renderIsTerminal = isatty.IsTerminal + const renderChangelogTemplate = "\n\n" + "# Please enter a changelog entry message. The text will be used\n" + "# unmodified for the changelog entry (other than the first line\n" + @@ -126,6 +130,10 @@ func prefillRenderChanges(template, changes string, autorelease bool) string { } func editRenderMessage(env *azldev.Env, specPath, template string) (string, error) { + if !renderIsTerminal(os.Stdin.Fd()) || !renderIsTerminal(os.Stdout.Fd()) { + return "", errors.New("non-interactive renders must use '--changelog-comment' and/or '--no-git-commit'") + } + args, err := renderEditor(env) if err != nil { return "", err diff --git a/internal/app/azldev/cmds/component/render_message_internal_test.go b/internal/app/azldev/cmds/component/render_message_internal_test.go index 96518c5ff..b9a43898c 100644 --- a/internal/app/azldev/cmds/component/render_message_internal_test.go +++ b/internal/app/azldev/cmds/component/render_message_internal_test.go @@ -21,6 +21,11 @@ import ( func setRenderEditor(t *testing.T, env *testutils.TestEnv, editor string) { t.Helper() + original := renderIsTerminal + renderIsTerminal = func(_ uintptr) bool { return true } + + t.Cleanup(func() { renderIsTerminal = original }) + osEnv, ok := env.TestOSEnv.(*testctx.TestOSEnv) require.True(t, ok) osEnv.SetEnv("EDITOR", editor) @@ -59,6 +64,97 @@ func TestRenderEditorSelection(t *testing.T) { } } +func TestRenderMessageRequiresInteractiveTTY(t *testing.T) { + for _, testCase := range []struct { + name string + stdinTTY, stdoutTTY bool + }{ + {name: "stdin is not a tty", stdoutTTY: true}, + {name: "stdout is not a tty", stdinTTY: true}, + {name: "neither is a tty"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setRenderEditor(t, env, "unavailable-editor") + + renderIsTerminal = func(descriptor uintptr) bool { + switch descriptor { + case os.Stdin.Fd(): + return testCase.stdinTTY + case os.Stdout.Fd(): + return testCase.stdoutTTY + default: + t.Fatalf("unexpected terminal descriptor: %d", descriptor) + + return false + } + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("must not invoke an editor without both TTYs: %v", cmd.Args) + + return nil + } + message, err := editRenderMessage(env.Env, "/working/curl.spec", renderLocalCommitTemplate) + require.ErrorContains(t, err, "non-interactive renders must use '--changelog-comment' and/or '--no-git-commit'") + assert.Empty(t, message) + }) + } +} + +func TestNonInteractiveRenderMessageOptions(t *testing.T) { + for _, testCase := range []struct { + name, release string + local, noGit, provided bool + wantError bool + }{ + {name: "provided upstream static", release: "1", provided: true}, + {name: "provided upstream autorelease", release: "%autorelease", provided: true}, + {name: "provided local", release: "1", local: true, provided: true}, + {name: "local static no commit", release: "1", local: true, noGit: true}, + {name: "local autorelease no commit", release: "%autorelease", local: true, noGit: true}, + {name: "upstream autorelease no commit", release: "%autorelease", noGit: true}, + {name: "upstream static no commit still needs message", release: "1", noGit: true, wantError: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + original := renderIsTerminal + renderIsTerminal = func(_ uintptr) bool { return false } + + t.Cleanup(func() { renderIsTerminal = original }) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + const specPath = "/working/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, specPath, + []byte(renderTestSpec("1", testCase.release, "")), fileperms.PublicFile)) + + bumped := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + require.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) + + bumped = true + + return nil + } + + update := &locklessRenderUpdate{ + local: testCase.local, noGitCommit: testCase.noGit, subjectProvided: testCase.provided, + } + if testCase.provided { + update.subject = "Provided render message" + } + + _, err := finishLocklessSpec(env.Env, specPath, update, true, false) + if testCase.wantError { + require.ErrorContains(t, err, "non-interactive renders must use") + } else { + require.NoError(t, err) + } + + assert.Equal(t, !testCase.local && testCase.release == "1" && !testCase.wantError, bumped) + }) + } +} + func TestFinishLocklessSpecMessages(t *testing.T) { for _, testCase := range []struct { name, release, message, template string From 91807db1de473e99f6b19ab35b87216481de2a46 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 6 Nov 2026 19:54:54 +0000 Subject: [PATCH 42/70] fix(component): reject non-increasing lockless render NEVRs Before removing existing lockless output, query the existing and prepared source-package NEVRs with host rpmspec, using each spec's own directory for macro resolution. Require the same package name and a strictly higher EVR under RPM version ordering. Report both NEVRs when rejecting equal or lower versions. Query failures also abort validation. Return a failing exit status regardless of --fail-on-error, and leave existing output untouched on validation errors. First renders, locked renders, check-only, and dry runs retain their existing behavior. No-change rebuilds do not bypass the NEVR requirement. Update user documentation, generated help, emitted guidance, and tests. Builds, unit tests, complete quality checks, and native RPM/macro smoke tests passed. Docker scenarios and snapshot updates remain blocked by host Docker-provider initialization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 14 +- .../reference/cli/azldev_component_render.md | 1 + .../app/azldev/agentskill/agentskill_test.go | 2 + .../azldev/agentskill/content/azldev.md.tmpl | 5 + .../content/refresh-upstream-commit.md.tmpl | 5 + .../content/withoutlockfile/azldev.md.tmpl | 5 + internal/app/azldev/cmds/component/render.go | 13 +- .../cmds/component/render_internal_test.go | 2 + .../azldev/cmds/component/render_lockless.go | 56 +++++++ .../render_lockless_internal_test.go | 11 +- .../component/render_output_internal_test.go | 147 ++++++++++++++++++ 11 files changed, 255 insertions(+), 6 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index aeddd0c7f..d5a01d83e 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -348,7 +348,19 @@ adjustment are skipped. The spec is **not** preprocessed with `rpmautospec proce No mock configuration is needed for lockless rendering. Comparison ignores the root `.git` entry without traversing or removing it; the working checkout retains its Git metadata until staging cleanup. -After comparison and the final release bump, lockless rendering removes the +After comparison and the final release bump, but immediately before removing +existing output, lockless rendering queries the existing and new source-package +NEVRs with host `rpmspec`. Queries expand each spec's macros using its own dist-git +directory. The package name must match, and the new epoch/version/release must +be strictly higher under RPM version ordering. Equal or lower NEVRs fail with +both values in the error message; query failures also abort. These errors return +a failing exit status even without `--fail-on-error`, and leave existing output +untouched. First renders with no existing directory skip the check. +`--check-only` and dry runs do not perform this replacement-time check. +`--no-change-rebuild` does not bypass it: a timestamp-only autorelease rebuild +still fails if its expanded NEVR has not increased. + +After successful validation, lockless rendering removes the existing output directory and moves all prepared entries except `.git` into its replacement. Files found only in the old output are not retained. Moves use filesystem renames, so the project work directory and rendered diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index d77f310bb..0d5eb30f1 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -36,6 +36,7 @@ Autorelease skips bumpspec. Missing messages are collected in an editor before bumping or replacing output; the final Git commit never prompts. Autorelease with --no-git-commit needs no message and never prompts. Message prompts require both stdin and stdout to be interactive TTYs. +Before replacing existing lockless output, require a higher NEVR using host rpmspec. The first commit also includes all pending changes outside the rendered output directory. Later commits are component-only. --no-git-commit skips staging and committing. Output must be inside the project repository; configure Git author identity and diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 1938002c2..2bf5522a2 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -218,6 +218,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") + assert.Contains(t, doc, "The new NEVR must be higher for the same package") assert.Contains(t, doc, "`--changelog-comment`") assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") assert.Contains(t, doc, "`$EDITOR` or a local editor") @@ -272,6 +273,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") + assert.Contains(t, doc, "The new NEVR must be higher for the same package") assert.Contains(t, doc, "`--changelog-comment`") assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") assert.Contains(t, doc, "`$EDITOR` or a local editor") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 75d124fad..94b330263 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -69,6 +69,11 @@ Lockless failures never write `RENDER_FAILED`. Before replacement, output is untouched; failures after replacement starts clear the component directory and use Git to restore committed output, or leave no directory if none existed. Recovery discards uncommitted changes in that directory and keeps no backups. +Before removing existing lockless output, host `rpmspec` compares source-package NEVRs. +The new NEVR must be higher for the same package; equal/lower values fail with both NEVRs shown. +Validation errors leave existing output untouched and fail even without `--fail-on-error`. +First renders, `--check-only`, and dry runs skip this replacement-time check. +`--no-change-rebuild` never bypasses the NEVR check. It never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. It retains all prepared dist-git files except `.git`, without file filtering or `spectool` queries, and replaces and diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 158b8c5cd..0962cdd10 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -110,6 +110,11 @@ Lockless failures never write `RENDER_FAILED`. Before replacement, output is untouched; failures after replacement starts clear the component directory and use Git to restore committed output, or leave no directory if none existed. Recovery discards uncommitted changes in that directory and keeps no backups. +Before removing existing lockless output, host `rpmspec` compares source-package NEVRs. +The new NEVR must be higher for the same package; equal/lower values fail with both NEVRs shown. +Validation errors leave existing output untouched and fail even without `--fail-on-error`. +First renders, `--check-only`, and dry runs skip this replacement-time check. +`--no-change-rebuild` never bypasses the NEVR check. The comparison ignores the root `.git` without traversing or removing it. Changelog handling precedes release preparation and is independent of version diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 226bbdbc0..3764044f4 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -64,6 +64,11 @@ Lockless failures never write `RENDER_FAILED`. Before replacement, output is untouched; failures after replacement starts clear the component directory and use Git to restore committed output, or leave no directory if none existed. Recovery discards uncommitted changes in that directory and keeps no backups. +Before removing existing lockless output, host `rpmspec` compares source-package NEVRs. +The new NEVR must be higher for the same package; equal/lower values fail with both NEVRs shown. +Validation errors leave existing output untouched and fail even without `--fail-on-error`. +First renders, `--check-only`, and dry runs skip this replacement-time check. +`--no-change-rebuild` never bypasses the NEVR check. It replaces and automatically commits the component output directory. The first commit also includes all staged changes, unstaged tracked changes, and non-ignored untracked files outside the rendered output directory. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index d2c143642..7ab1f0251 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -68,7 +68,8 @@ const renderMessageHelp = `Non-autorelease specs use rpmdev-bumpspec -c with --c Autorelease skips bumpspec. Missing messages are collected in an editor before bumping or replacing output; the final Git commit never prompts. Autorelease with --no-git-commit needs no message and never prompts. -Message prompts require both stdin and stdout to be interactive TTYs.` +Message prompts require both stdin and stdout to be interactive TTYs. +Before replacing existing lockless output, require a higher NEVR using host rpmspec.` // NewRenderCmd constructs a [cobra.Command] for the "component render" CLI subcommand. func NewRenderCmd() *cobra.Command { @@ -385,7 +386,7 @@ func sortRenderResults(results []*RenderResult) { }) } -// checkRenderErrors reports no-change refusals regardless of failOnError, and +// checkRenderErrors reports no-change refusals and NEVR validation errors regardless of failOnError, and // other error or cancelled results only when failOnError is set. func checkRenderErrors(results []*RenderResult, failOnError bool) error { var ( @@ -398,7 +399,7 @@ func checkRenderErrors(results []*RenderResult, failOnError bool) error { continue } - if errors.Is(result.renderErr, errNoRenderChanges) { + if errors.Is(result.renderErr, errNoRenderChanges) || errors.Is(result.renderErr, errRenderNEVRValidation) { noChangeErrors = append(noChangeErrors, fmt.Errorf("%#q: %w", result.Component, result.renderErr)) } @@ -993,6 +994,12 @@ func moveRenderedOutput(env *azldev.Env, tempDir string, prep *preparedComponent componentOutputDir) } + if exists && prep.lockless { + if err := requireRenderNEVRIncrease(env, tempDir, prep); err != nil { + return err + } + } + prep.outputExisted = exists prep.outputTouched = true diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 2af724eed..9f96d078e 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -730,6 +730,7 @@ func TestFinishComponentRender_PreservesGitMetadata(t *testing.T) { } mockResults := map[string]*sources.ComponentMockResult{"curl": {}} + stubIncreasingRenderNEVR(t, env) changed, err := finishComponentRender(env.Env, prep, mockResults, "/staging", true, testCase.checkOnly, false) require.NoError(t, err) assert.False(t, changed, "Git metadata alone must not count as rendered drift") @@ -845,6 +846,7 @@ func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { lockless: true, update: &locklessRenderUpdate{subject: "updated curl"}, } + stubIncreasingRenderNEVR(t, env) result := finishOneComponent(env.Env, prep, map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, testCase.checkOnly, testCase.noChangeRebuild) diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 128fef366..3bb0f1ccd 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -24,6 +24,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/lockfile" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" @@ -39,6 +40,8 @@ var errLocalManualRebuild = errors.New( "local non-autorelease components must manually update their release and changelog before rendering again", ) +var errRenderNEVRValidation = errors.New("lockless render NEVR validation failed") + type locklessRenderUpdate struct { repoRoot string path string @@ -893,6 +896,59 @@ func locklessSpecUsesAutorelease(env *azldev.Env, path string) (bool, error) { return sources.ReleaseUsesAutorelease(release), nil } +func requireRenderNEVRIncrease(env *azldev.Env, preparedDir string, prep *preparedComponent) error { + existing, err := queryRenderSpecNEVR(env, filepath.Join(prep.compOutputDir, prep.specFilename)) + if err != nil { + return fmt.Errorf("%w: querying existing NEVR:\n%w", errRenderNEVRValidation, err) + } + + rendered, err := queryRenderSpecNEVR(env, filepath.Join(preparedDir, prep.specFilename)) + if err != nil { + return fmt.Errorf("%w: querying new NEVR:\n%w", errRenderNEVRValidation, err) + } + + if existing.Name != rendered.Name || rendered.Version.Compare(&existing.Version) <= 0 { + return fmt.Errorf("%w: new NEVR must be higher than existing NEVR for the same package; "+ + "existing %#q; new %#q", errRenderNEVRValidation, + existing.Name+"-"+existing.Version.String(), rendered.Name+"-"+rendered.Version.String()) + } + + return nil +} + +func queryRenderSpecNEVR(env *azldev.Env, path string) (*rpm.SpecInfo, error) { + if err := prereqs.RequireExecutable(env, "rpmspec", nil); err != nil { + return nil, fmt.Errorf("lockless render NEVR validation requires 'rpmspec':\n%w", err) + } + + dir := filepath.Dir(path) + raw := exec.CommandContext(env, "rpmspec", "-q", "--srpm", + "--define", "_sourcedir "+dir, "--define", "_specdir "+dir, + "--queryformat", rpm.SrpmQueryFormat, path) + raw.Dir = dir + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + command, err := env.Command(raw) + if err != nil { + return nil, fmt.Errorf("creating NEVR query for %#q:\n%w", path, err) + } + + output, err := command.RunAndGetOutput(env) + if err != nil { + return nil, fmt.Errorf("querying NEVR for %#q (stderr: %#q):\n%w", path, stderr.String(), err) + } + + info, err := rpm.ParseSrpmQueryOutput(path, output) + if err != nil { + return nil, fmt.Errorf("parsing NEVR query for %#q:\n%w", path, err) + } + + return info, nil +} + func writeNoChangeRebuild(env *azldev.Env, dir string) error { if env.DryRun() { return nil diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index 1d2f5bb6d..ba55bfe85 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -1272,6 +1272,14 @@ func assertCurlRenderCommit(t *testing.T, args []string, message string, provide assert.Equal(t, wantArgs, args) } +func finishPreparedCurlRender(t *testing.T, env *testutils.TestEnv, prep *preparedComponent) (bool, error) { + t.Helper() + stubIncreasingRenderNEVR(t, env) + + return finishComponentRender(env.Env, prep, + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, false, false) +} + func TestPrepareComponentSources_ChangelogAndReleaseBeforeOverlays(t *testing.T) { const releaseTag = "Release: " @@ -1441,8 +1449,7 @@ upstream-commit = "aaaaaaa" assert.False(t, bumped) prep.compOutputDir = "/project/SPECS/c/curl" - changed, err := finishComponentRender(env.Env, prep, - map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, false, false) + changed, err := finishPreparedCurlRender(t, env, prep) require.NoError(t, err) assert.True(t, changed) assert.True(t, bumped) diff --git a/internal/app/azldev/cmds/component/render_output_internal_test.go b/internal/app/azldev/cmds/component/render_output_internal_test.go index ce6e47279..29a850170 100644 --- a/internal/app/azldev/cmds/component/render_output_internal_test.go +++ b/internal/app/azldev/cmds/component/render_output_internal_test.go @@ -16,6 +16,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/dirdiff" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" @@ -35,6 +36,150 @@ const ( renderTestSpecDiff = "SPECS/c/curl/curl.spec\n" ) +func TestLocklessRenderNEVRBeforeReplacement(t *testing.T) { + for _, testCase := range []struct { + name, existing, rendered, newName string + reject, queryFails, toolMissing bool + }{ + {name: "numeric release increase", existing: "1-9", rendered: "1-10"}, + {name: "equal", existing: "1-9", rendered: "1-9", reject: true}, + {name: "version downgrade", existing: "2-1", rendered: "1-99", reject: true}, + {name: "epoch increase", existing: "9-9", rendered: "1:1-1"}, + {name: "epoch decrease", existing: "1:1-1", rendered: "9-9", reject: true}, + {name: "prerelease downgrade", existing: "1-1", rendered: "1~rc1-2", reject: true}, + {name: "name change", existing: "1-1", rendered: "2-1", newName: "other", reject: true}, + {name: "query failure", existing: "1-1", rendered: "2-1", queryFails: true, reject: true}, + {name: "missing rpmspec", existing: "1-1", rendered: "2-1", toolMissing: true, reject: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, true) + + if !testCase.toolMissing { + env.CmdFactory.RegisterCommandInSearchPath("rpmspec") + } + + prep := &preparedComponent{ + lockless: true, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, + } + existing, err := rpm.NewVersion(testCase.existing) + require.NoError(t, err) + rendered, err := rpm.NewVersion(testCase.rendered) + require.NoError(t, err) + + newName := "curl" + if testCase.newName != "" { + newName = testCase.newName + } + + queries := 0 + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + queries++ + + assert.False(t, prep.outputTouched, "NEVR checks must precede removal and recovery state") + assertOriginalRenderOutput(t, env.TestFS, true) + + dir := testRenderOutputDir + + name, version := "curl", existing + if queries == 2 { + dir, name, version = "/staging/curl", newName, rendered + } + + assert.Equal(t, dir, cmd.Dir) + assert.Equal(t, []string{ + "rpmspec", "-q", "--srpm", "--define", "_sourcedir " + dir, + "--define", "_specdir " + dir, "--queryformat", rpm.SrpmQueryFormat, + filepath.Join(dir, "curl.spec"), + }, cmd.Args) + + if testCase.queryFails { + return "", errors.New("rpmspec failed") + } + + return "name=" + name + "\nepoch=" + strconv.Itoa(version.Epoch()) + + "\nversion=" + version.Version() + "\nrelease=" + version.Release() + "\n", nil + } + + err = moveRenderedOutput(env.Env, "/staging/curl", prep, true) + if testCase.reject { + require.ErrorIs(t, err, errRenderNEVRValidation) + assertOriginalRenderOutput(t, env.TestFS, true) + assert.False(t, prep.outputTouched) + + if !testCase.queryFails && !testCase.toolMissing { + require.ErrorContains(t, err, "existing `curl-"+testCase.existing+"`") + require.ErrorContains(t, err, "new `"+newName+"-"+testCase.rendered+"`") + } + + for _, failOnError := range []bool{false, true} { + require.ErrorIs(t, checkRenderErrors([]*RenderResult{{ + Component: "curl", Status: renderStatusError, renderErr: err, + }}, failOnError), errRenderNEVRValidation) + } + } else { + require.NoError(t, err) + assert.True(t, prep.outputTouched) + + content, readErr := fileutils.ReadFile(env.TestFS, filepath.Join(testRenderOutputDir, "sidecar")) + require.NoError(t, readErr) + assert.Equal(t, "new sidecar", string(content)) + } + + switch { + case testCase.toolMissing: + assert.Zero(t, queries) + case testCase.queryFails: + assert.Equal(t, 1, queries) + default: + assert.Equal(t, 2, queries) + } + }) + } +} + +func TestRenderNEVRSkipsFirstAndLockedRenders(t *testing.T) { + for _, existing := range []bool{false, true} { + t.Run(strconv.FormatBool(existing), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, existing) + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("must not query NEVR for first or locked output: %v", cmd.Args) + + return "", nil + } + require.NoError(t, moveRenderedOutput(env.Env, "/staging/curl", &preparedComponent{ + lockless: !existing, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, + }, true)) + }) + } +} + +func stubIncreasingRenderNEVR(t *testing.T, env *testutils.TestEnv) { + t.Helper() + + env.CmdFactory.RegisterCommandInSearchPath("rpmspec") + original := env.CmdFactory.RunAndGetOutputHandler + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[0] == "rpmspec" { + release := "1" + if cmd.Dir == "/staging/curl" { + release = "2" + } + + return "name=curl\nepoch=0\nversion=1\nrelease=" + release + "\n", nil + } + + if original != nil { + return original(cmd) + } + + t.Fatalf("unexpected command: %v", cmd.Args) + + return "", nil + } +} + func writeOriginalRenderOutput(t *testing.T, fileSystem afero.Fs, root string) { t.Helper() @@ -128,6 +273,7 @@ func TestMoveLocklessOutput_PublicationFailure(t *testing.T) { mockResult.Error = errors.New("injected processing failure") } + stubIncreasingRenderNEVR(t, env) result := finishOneComponent(env.Env, prep, map[string]*sources.ComponentMockResult{"curl": mockResult}, "/staging", failure != renderTestOverwrite, false, false) @@ -167,6 +313,7 @@ func TestCommitLocklessRenders_RestoresFailedOutput(t *testing.T) { env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { return nil } + stubIncreasingRenderNEVR(t, env) result := finishOneComponent(env.Env, prepared[0], map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, false, false) require.NoError(t, result.renderErr) From 63674c090350afc6724a8edc0af2bd0b07118e81 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 6 Nov 2026 20:33:37 +0000 Subject: [PATCH 43/70] fix(component): isolate lockless bumpspec RPM context Run lockless rpmdev-bumpspec with a temporary PATH-based RPM wrapper. Set _sourcedir and _specdir to the staged spec directory while preserving HOME, packager settings, locale, timezone, and the user's RPM macros. Reuse the existing wrapper generator and remove each private wrapper on success and failure. Treat cleanup failures as render errors. Add focused environment, isolation, and filesystem-failure tests and update guidance. Validation: build and full unit tests passed via mage all; mage check all and mage docs passed. Native RPM fixtures passed for quoted paths, sidecar macros, environment preservation, and success/failure cleanup. Docker scenarios remain blocked by the existing XDG_RUNTIME_DIR provider error. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 6 + .../app/azldev/agentskill/agentskill_test.go | 10 ++ .../azldev/agentskill/content/azldev.md.tmpl | 4 +- .../content/withoutlockfile/azldev.md.tmpl | 3 + .../azldev/cmds/component/render_lockless.go | 36 ++++- .../render_lockless_internal_test.go | 153 +++++++++++++++++- internal/app/azldev/core/sources/bumpspec.go | 6 +- 7 files changed, 212 insertions(+), 6 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index d5a01d83e..0d10e19be 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -306,6 +306,12 @@ before output replacement. Missing-comment finishing is serialized to prevent simultaneous editors. Local components and autorelease specs skip `rpmdev-bumpspec` entirely. +For upstream non-autorelease specs, a temporary process-local RPM wrapper sets +`_sourcedir` and `_specdir` to the spec's directory so sidecar macros resolve +correctly. Rendering preserves your `HOME`, packager settings, locale, and +timezone; it does not edit your `.rpmmacros`. The wrapper is removed after the +invocation, including when bumpspec fails. + When `--changelog-comment` is omitted, the prompt depends on how the message is used: | Component | Automatic Git commit | `--no-git-commit` | diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 2bf5522a2..b3e9b2ec9 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -191,6 +191,16 @@ func TestSkillsDocumentDistGitRepo(t *testing.T) { } } +func TestSkillsDocumentLocklessRPMContext(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "temporary process-local RPM wrapper sets `_sourcedir` and `_specdir`") + assert.Contains(t, doc, "without changing `HOME` or editing the user's `.rpmmacros`") + assert.Contains(t, doc, "removed after the invocation") + } +} + func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { for _, withoutLockfile := range []bool{false, true} { doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev", testParams()) diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 94b330263..47a2cbcd2 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -55,7 +55,9 @@ then writes macros, applies overlays, and updates `sources`. Version detection and carryover use the upstream spec before overlays. Overlays can override the prepared release and changelog. Non-autorelease rendering then runs host `rpmdev-bumpspec`, without synthetic history or its legacy -release adjustment. Lockless rendering +release adjustment. A temporary process-local RPM wrapper sets `_sourcedir` and `_specdir` +to the spec directory without changing `HOME` or editing the user's `.rpmmacros`; +it is removed after the invocation, including on failure. Lockless rendering preserves the existing spaces and tabs after `Release:` when editing its value, and compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless renders require `--no-change-rebuild`; otherwise rendering errors diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 3764044f4..801fbd67e 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -51,6 +51,9 @@ then writes macros, applies overlays, and updates `sources`. Version detection and carryover use the upstream spec before overlays. Overlays can override the prepared release and changelog. Non-autorelease rendering then runs host `rpmdev-bumpspec`. +A temporary process-local RPM wrapper sets `_sourcedir` and `_specdir` to the spec directory +without changing `HOME` or editing the user's `.rpmmacros`; it is removed after the invocation, +including on failure. Release edits preserve the existing spaces and tabs after `Release:`. Rendering compares output before the final release bump; `--check-only` stops before that bump. Unchanged lockless renders require `--no-change-rebuild`; otherwise rendering errors diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 3bb0f1ccd..c0c8a19ab 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "log/slog" + "os" "os/exec" "path/filepath" "regexp" @@ -983,7 +984,7 @@ func writeNoChangeRebuild(env *azldev.Env, dir string) error { return nil } -func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate) (bool, error) { +func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate) (changed bool, err error) { before, err := fileutils.ReadFile(env.FS(), path) if err != nil { return false, fmt.Errorf("reading spec before release bump:\n%w", err) @@ -1010,6 +1011,15 @@ func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate raw := exec.CommandContext(env, sources.RPMDevBumpspecBinary, "-c", update.subject, path) raw.Dir = filepath.Dir(path) + wrapperDir, err := prepareRenderBumpspecEnvironment(env, raw, path) + if wrapperDir != "" { + defer fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), wrapperDir, &err) + } + + if err != nil { + return false, err + } + var stderr bytes.Buffer raw.Stderr = &stderr @@ -1031,6 +1041,30 @@ func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate return !bytes.Equal(before, after), nil } +func prepareRenderBumpspecEnvironment(env *azldev.Env, raw *exec.Cmd, specPath string) (string, error) { + dir, err := fileutils.MkdirTempInTempDir(env.FS(), "azldev-render-rpm-") + if err != nil { + return "", fmt.Errorf("creating temporary render RPM wrapper directory:\n%w", err) + } + + if strings.ContainsRune(dir, os.PathListSeparator) { + return dir, fmt.Errorf("temporary RPM wrapper directory %#q contains the PATH separator", dir) + } + + wrapper := sources.GenerateRPMDevBumpspecWrapper(sources.RPMDevBumpspecRequest{SpecPath: specPath}) + if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "rpm"), + []byte(wrapper), fileperms.PublicExecutable); err != nil { + return dir, fmt.Errorf("writing temporary render RPM wrapper:\n%w", err) + } + + originalPath := env.OSEnv().Getenv("PATH") + raw.Env = append(raw.Environ(), + "PATH="+dir+string(os.PathListSeparator)+originalPath, + "RPMDEV_BUMPSPEC_ORIGINAL_PATH="+originalPath) + + return dir, nil +} + func commitLocklessRenders( env *azldev.Env, prepared []*preparedComponent, results []*RenderResult, checkOnly bool, ) { diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index ba55bfe85..f9a0d5f5c 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "os" "os/exec" "path/filepath" "slices" @@ -34,8 +35,9 @@ import ( ) const ( - renderTestRevParse = "rev-parse" - renderTestProjectRoot = "/project" + renderTestRevParse = "rev-parse" + renderTestProjectRoot = "/project" + renderTestWrapperCleanupFailure = "cleanup" ) func renderTestSpec(version, release, changelog string) string { @@ -1694,6 +1696,153 @@ func TestBumpLocklessSpec_Failures(t *testing.T) { } } +func TestBumpLocklessSpec_ProcessLocalRPMContext(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(strconv.FormatBool(fail), func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + const path = "/working/spec dir's/curl.spec" + + before := renderTestSpec("1", "1", "") + after := renderTestSpec("1", "2", "- Render message\n") + + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(before), fileperms.PublicFile)) + + var wrapperDir string + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + assert.Equal(t, []string{"rpmdev-bumpspec", "-c", "Render message", path}, cmd.Args) + assert.Equal(t, filepath.Dir(path), cmd.Dir) + wrapperDir = filepath.SplitList(renderCommandEnvValue(cmd.Env, "PATH"))[0] + wrapper, err := fileutils.ReadFile(env.TestFS, filepath.Join(wrapperDir, "rpm")) + require.NoError(t, err) + assert.Contains(t, string(wrapper), "'_sourcedir /working/spec dir'\"'\"'s'") + assert.Contains(t, string(wrapper), "PATH=\"$RPMDEV_BUMPSPEC_ORIGINAL_PATH\"") + assert.Equal(t, env.OSEnv().Getenv("PATH"), + renderCommandEnvValue(cmd.Env, "RPMDEV_BUMPSPEC_ORIGINAL_PATH")) + + inherited := (&exec.Cmd{}).Environ() + for _, key := range []string{"HOME", "LC_ALL", "TZ"} { + assert.Equal(t, renderCommandEnvValue(inherited, key), renderCommandEnvValue(cmd.Env, key)) + } + + if fail { + return errors.New("bumpspec failed") + } + + return fileutils.WriteFile(env.TestFS, path, []byte(after), fileperms.PublicFile) + } + + changed, err := bumpLocklessSpec(env.Env, path, &locklessRenderUpdate{subject: "Render message"}) + if fail { + require.ErrorContains(t, err, "bumpspec failed") + assert.False(t, changed) + } else { + require.NoError(t, err) + assert.True(t, changed) + } + + require.NotEmpty(t, wrapperDir) + exists, err := fileutils.Exists(env.TestFS, wrapperDir) + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + +func renderCommandEnvValue(values []string, key string) string { + for _, value := range slices.Backward(values) { + if text, ok := strings.CutPrefix(value, key+"="); ok { + return text + } + } + + return "" +} + +func TestPrepareRenderBumpspecEnvironment_IsolatedInvocations(t *testing.T) { + env := testutils.NewTestEnv(t) + first := &exec.Cmd{Env: []string{"HOME=/user", "LC_ALL=custom", "TZ=custom", "KEEP=value"}} + second := &exec.Cmd{} + firstDir, err := prepareRenderBumpspecEnvironment(env.Env, first, "/first/curl.spec") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, env.TestFS.RemoveAll(firstDir)) }) + + secondDir, err := prepareRenderBumpspecEnvironment(env.Env, second, "/second/curl.spec") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, env.TestFS.RemoveAll(secondDir)) }) + + assert.NotEqual(t, firstDir, secondDir) + + for _, item := range []string{"HOME=/user", "LC_ALL=custom", "TZ=custom", "KEEP=value"} { + assert.Contains(t, first.Env, item) + } + + require.NoError(t, env.TestFS.RemoveAll(firstDir)) + + wrapper, err := fileutils.ReadFile(env.TestFS, filepath.Join(secondDir, "rpm")) + require.NoError(t, err) + assert.Contains(t, string(wrapper), "'_sourcedir /second'") + assert.NotContains(t, string(wrapper), "'_sourcedir /first'") +} + +type renderBumpspecFaultFS struct { + afero.Fs + failure string + dir string +} + +func (fs *renderBumpspecFaultFS) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + if filepath.Base(name) == "rpm" { + fs.dir = filepath.Dir(name) + if fs.failure == "write" { + return nil, errors.New("injected wrapper write failure") + } + } + + return fs.Fs.OpenFile(name, flag, perm) +} + +func (fs *renderBumpspecFaultFS) RemoveAll(path string) error { + if path == fs.dir && fs.failure == renderTestWrapperCleanupFailure { + return errors.New("injected wrapper cleanup failure") + } + + return fs.Fs.RemoveAll(path) +} + +func TestBumpLocklessSpec_WrapperFilesystemFailures(t *testing.T) { + for _, failure := range []string{"write", renderTestWrapperCleanupFailure} { + t.Run(failure, func(t *testing.T) { + env := testutils.NewTestEnv(t) + faultFS := &renderBumpspecFaultFS{Fs: env.TestFS, failure: failure} + env.TestFS = faultFS + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + path := "/working/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, path, + []byte(renderTestSpec("1", "1", "")), fileperms.PublicFile)) + + called := false + env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + called = true + + return fileutils.WriteFile(env.TestFS, path, + []byte(renderTestSpec("1", "2", "")), fileperms.PublicFile) + } + + _, err := bumpLocklessSpec(env.Env, path, &locklessRenderUpdate{subject: "Render message"}) + require.ErrorContains(t, err, "injected wrapper "+failure+" failure") + assert.Equal(t, failure == renderTestWrapperCleanupFailure, called) + require.NotEmpty(t, faultFS.dir) + exists, err := fileutils.Exists(env.TestFS, faultFS.dir) + require.NoError(t, err) + assert.Equal(t, failure == renderTestWrapperCleanupFailure, exists) + }) + } +} + func TestCommitLocklessRenders_CheckOnlyAndFailure(t *testing.T) { for _, checkOnly := range []bool{false, true} { t.Run(strconv.FormatBool(checkOnly), func(t *testing.T) { diff --git a/internal/app/azldev/core/sources/bumpspec.go b/internal/app/azldev/core/sources/bumpspec.go index ee1a54bf5..5f4da98ad 100644 --- a/internal/app/azldev/core/sources/bumpspec.go +++ b/internal/app/azldev/core/sources/bumpspec.go @@ -206,7 +206,7 @@ func prepareRPMDevBumpspecHome(ctx opctx.Ctx, request RPMDevBumpspecRequest) err return fmt.Errorf("failed to create isolated RPM wrapper directory %#q:\n%w", filepath.Dir(rpmWrapperPath), err) } - if err := fileutils.WriteFile(ctx.FS(), rpmWrapperPath, []byte(rpmdevBumpspecWrapper(request)), + if err := fileutils.WriteFile(ctx.FS(), rpmWrapperPath, []byte(GenerateRPMDevBumpspecWrapper(request)), fileperms.PublicExecutable); err != nil { return fmt.Errorf("failed to create isolated RPM wrapper %#q:\n%w", rpmWrapperPath, err) } @@ -214,7 +214,9 @@ func prepareRPMDevBumpspecHome(ctx opctx.Ctx, request RPMDevBumpspecRequest) err return nil } -func rpmdevBumpspecWrapper(request RPMDevBumpspecRequest) string { +// GenerateRPMDevBumpspecWrapper generates an RPM wrapper with request-local spec +// and build context. RPMDEV_BUMPSPEC_ORIGINAL_PATH must contain the original PATH. +func GenerateRPMDevBumpspecWrapper(request RPMDevBumpspecRequest) string { args := append([]string{"rpm"}, rpmdevBumpspecContextArgs(request)...) quoted := make([]string, len(args)) From 2d3cedf025a01cf8706496dc90f15685dd5a435a Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Fri, 6 Nov 2026 22:42:31 +0000 Subject: [PATCH 44/70] fix(component): allow equal NEVRs for autorelease renders Allow a new lockless render's queried NEVR to equal the existing NEVR when the final staged spec uses autorelease. Continue requiring a strictly higher NEVR for non-autorelease specs and reject all downgrades and package-name changes. First renders still skip the comparison. Cover equality, increases, downgrades, name changes, and first renders in tests. Update CLI help, generated docs, and emitted agent guidance. Validation: mage fix all, mage unit, mage check all, and mage docs passed. Native render fixtures passed for local/upstream autorelease rebuilds and downgrade rejection. Docker scenarios and snapshot regeneration remain blocked by the existing XDG_RUNTIME_DIR provider error. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 12 ++-- .../reference/cli/azldev_component_render.md | 3 +- .../app/azldev/agentskill/agentskill_test.go | 6 +- .../azldev/agentskill/content/azldev.md.tmpl | 3 +- .../content/refresh-upstream-commit.md.tmpl | 3 +- .../content/withoutlockfile/azldev.md.tmpl | 3 +- internal/app/azldev/cmds/component/render.go | 5 +- .../cmds/component/render_internal_test.go | 6 +- .../azldev/cmds/component/render_lockless.go | 23 +++++-- .../component/render_output_internal_test.go | 61 +++++++++++++++---- 10 files changed, 93 insertions(+), 32 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 0d10e19be..c8f41b791 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -357,14 +357,18 @@ the working checkout retains its Git metadata until staging cleanup. After comparison and the final release bump, but immediately before removing existing output, lockless rendering queries the existing and new source-package NEVRs with host `rpmspec`. Queries expand each spec's macros using its own dist-git -directory. The package name must match, and the new epoch/version/release must -be strictly higher under RPM version ordering. Equal or lower NEVRs fail with -both values in the error message; query failures also abort. These errors return +directory. The package name must match. When the final new spec uses +`%autorelease`, its new epoch/version/release must be equal to or higher than the +existing value under RPM version ordering. Non-autorelease specs still require +a strictly higher value. Autorelease permits equality because a subsequent Git +commit can advance its Git-derived release; the query does not calculate that +future release. Lower NEVRs, and equal non-autorelease NEVRs, fail with both values +in the error message; query failures also abort. These errors return a failing exit status even without `--fail-on-error`, and leave existing output untouched. First renders with no existing directory skip the check. `--check-only` and dry runs do not perform this replacement-time check. `--no-change-rebuild` does not bypass it: a timestamp-only autorelease rebuild -still fails if its expanded NEVR has not increased. +may have an equal expanded NEVR, but must not decrease it. After successful validation, lockless rendering removes the existing output directory and moves all prepared entries except `.git` into diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 0d5eb30f1..425d5a1f1 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -36,7 +36,8 @@ Autorelease skips bumpspec. Missing messages are collected in an editor before bumping or replacing output; the final Git commit never prompts. Autorelease with --no-git-commit needs no message and never prompts. Message prompts require both stdin and stdout to be interactive TTYs. -Before replacing existing lockless output, require a higher NEVR using host rpmspec. +Before replacing existing lockless output, compare NEVRs using host rpmspec. +Autorelease requires an equal or higher NEVR; non-autorelease requires a higher NEVR. The first commit also includes all pending changes outside the rendered output directory. Later commits are component-only. --no-git-commit skips staging and committing. Output must be inside the project repository; configure Git author identity and diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index b3e9b2ec9..5fdca8cbf 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -228,7 +228,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") - assert.Contains(t, doc, "The new NEVR must be higher for the same package") + assert.Contains(t, doc, "The new NEVR must be equal or higher for a final autorelease spec") + assert.Contains(t, doc, "non-autorelease requires a higher NEVR") assert.Contains(t, doc, "`--changelog-comment`") assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") assert.Contains(t, doc, "`$EDITOR` or a local editor") @@ -283,7 +284,8 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") - assert.Contains(t, doc, "The new NEVR must be higher for the same package") + assert.Contains(t, doc, "The new NEVR must be equal or higher for a final autorelease spec") + assert.Contains(t, doc, "non-autorelease requires a higher NEVR") assert.Contains(t, doc, "`--changelog-comment`") assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") assert.Contains(t, doc, "`$EDITOR` or a local editor") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 47a2cbcd2..9a11c3b0d 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -72,7 +72,8 @@ untouched; failures after replacement starts clear the component directory and use Git to restore committed output, or leave no directory if none existed. Recovery discards uncommitted changes in that directory and keeps no backups. Before removing existing lockless output, host `rpmspec` compares source-package NEVRs. -The new NEVR must be higher for the same package; equal/lower values fail with both NEVRs shown. +The package name must match. The new NEVR must be equal or higher for a final autorelease spec; +non-autorelease requires a higher NEVR. Invalid values fail with both NEVRs shown. Validation errors leave existing output untouched and fail even without `--fail-on-error`. First renders, `--check-only`, and dry runs skip this replacement-time check. `--no-change-rebuild` never bypasses the NEVR check. diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 0962cdd10..0ef5fc999 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -111,7 +111,8 @@ untouched; failures after replacement starts clear the component directory and use Git to restore committed output, or leave no directory if none existed. Recovery discards uncommitted changes in that directory and keeps no backups. Before removing existing lockless output, host `rpmspec` compares source-package NEVRs. -The new NEVR must be higher for the same package; equal/lower values fail with both NEVRs shown. +The package name must match. The new NEVR must be equal or higher for a final autorelease spec; +non-autorelease requires a higher NEVR. Invalid values fail with both NEVRs shown. Validation errors leave existing output untouched and fail even without `--fail-on-error`. First renders, `--check-only`, and dry runs skip this replacement-time check. `--no-change-rebuild` never bypasses the NEVR check. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 801fbd67e..aa4ef1ce5 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -68,7 +68,8 @@ untouched; failures after replacement starts clear the component directory and use Git to restore committed output, or leave no directory if none existed. Recovery discards uncommitted changes in that directory and keeps no backups. Before removing existing lockless output, host `rpmspec` compares source-package NEVRs. -The new NEVR must be higher for the same package; equal/lower values fail with both NEVRs shown. +The package name must match. The new NEVR must be equal or higher for a final autorelease spec; +non-autorelease requires a higher NEVR. Invalid values fail with both NEVRs shown. Validation errors leave existing output untouched and fail even without `--fail-on-error`. First renders, `--check-only`, and dry runs skip this replacement-time check. `--no-change-rebuild` never bypasses the NEVR check. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 7ab1f0251..4bc676b6d 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -69,7 +69,8 @@ Autorelease skips bumpspec. Missing messages are collected in an editor before bumping or replacing output; the final Git commit never prompts. Autorelease with --no-git-commit needs no message and never prompts. Message prompts require both stdin and stdout to be interactive TTYs. -Before replacing existing lockless output, require a higher NEVR using host rpmspec.` +Before replacing existing lockless output, compare NEVRs using host rpmspec. +Autorelease requires an equal or higher NEVR; non-autorelease requires a higher NEVR.` // NewRenderCmd constructs a [cobra.Command] for the "component render" CLI subcommand. func NewRenderCmd() *cobra.Command { @@ -995,7 +996,7 @@ func moveRenderedOutput(env *azldev.Env, tempDir string, prep *preparedComponent } if exists && prep.lockless { - if err := requireRenderNEVRIncrease(env, tempDir, prep); err != nil { + if err := validateRenderNEVR(env, tempDir, prep); err != nil { return err } } diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 9f96d078e..e77de67bc 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -715,9 +715,11 @@ func TestFinishComponentRender_PreservesGitMetadata(t *testing.T) { comp.EXPECT().GetName().AnyTimes().Return("curl") for _, root := range []string{"/staging/curl", "/output/curl"} { - for _, name := range []string{"curl.spec", ".gitignore"} { + for name, content := range map[string]string{ + "curl.spec": "Name: curl\nVersion: 1\nRelease: 1\n", ".gitignore": "unchanged", + } { require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(root, name), - []byte("unchanged"), fileperms.PublicFile)) + []byte(content), fileperms.PublicFile)) } } diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index c0c8a19ab..257a66011 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -897,20 +897,33 @@ func locklessSpecUsesAutorelease(env *azldev.Env, path string) (bool, error) { return sources.ReleaseUsesAutorelease(release), nil } -func requireRenderNEVRIncrease(env *azldev.Env, preparedDir string, prep *preparedComponent) error { +func validateRenderNEVR(env *azldev.Env, preparedDir string, prep *preparedComponent) error { existing, err := queryRenderSpecNEVR(env, filepath.Join(prep.compOutputDir, prep.specFilename)) if err != nil { return fmt.Errorf("%w: querying existing NEVR:\n%w", errRenderNEVRValidation, err) } - rendered, err := queryRenderSpecNEVR(env, filepath.Join(preparedDir, prep.specFilename)) + newPath := filepath.Join(preparedDir, prep.specFilename) + + rendered, err := queryRenderSpecNEVR(env, newPath) if err != nil { return fmt.Errorf("%w: querying new NEVR:\n%w", errRenderNEVRValidation, err) } - if existing.Name != rendered.Name || rendered.Version.Compare(&existing.Version) <= 0 { - return fmt.Errorf("%w: new NEVR must be higher than existing NEVR for the same package; "+ - "existing %#q; new %#q", errRenderNEVRValidation, + autorelease, err := locklessSpecUsesAutorelease(env, newPath) + if err != nil { + return fmt.Errorf("%w: checking new spec autorelease:\n%w", errRenderNEVRValidation, err) + } + + requirement := "higher than" + if autorelease { + requirement = "equal to or higher than" + } + + comparison := rendered.Version.Compare(&existing.Version) + if existing.Name != rendered.Name || comparison < 0 || (comparison == 0 && !autorelease) { + return fmt.Errorf("%w: new NEVR must be %s existing NEVR for the same package; "+ + "existing %#q; new %#q", errRenderNEVRValidation, requirement, existing.Name+"-"+existing.Version.String(), rendered.Name+"-"+rendered.Version.String()) } diff --git a/internal/app/azldev/cmds/component/render_output_internal_test.go b/internal/app/azldev/cmds/component/render_output_internal_test.go index 29a850170..cbdbe457a 100644 --- a/internal/app/azldev/cmds/component/render_output_internal_test.go +++ b/internal/app/azldev/cmds/component/render_output_internal_test.go @@ -34,15 +34,30 @@ const ( renderTestCheckoutCommand = "checkout" renderTestOverwrite = "overwrite" renderTestSpecDiff = "SPECS/c/curl/curl.spec\n" + renderTestAutorelease = "%autorelease -b 9" ) func TestLocklessRenderNEVRBeforeReplacement(t *testing.T) { for _, testCase := range []struct { - name, existing, rendered, newName string - reject, queryFails, toolMissing bool + name, existing, rendered, newName, rawRelease string + reject, queryFails, toolMissing bool }{ {name: "numeric release increase", existing: "1-9", rendered: "1-10"}, {name: "equal", existing: "1-9", rendered: "1-9", reject: true}, + {name: "autorelease equal", existing: "1-9", rendered: "1-9", rawRelease: renderTestAutorelease}, + {name: "autorelease increase", existing: "1-9", rendered: "1-10", rawRelease: "%autorelease -b 10"}, + { + name: "autorelease release downgrade", existing: "1-10", rendered: "1-9", + rawRelease: renderTestAutorelease, reject: true, + }, + { + name: "autorelease version downgrade", existing: "2-1", rendered: "1-9", + rawRelease: renderTestAutorelease, reject: true, + }, + { + name: "autorelease name change", existing: "1-9", rendered: "1-9", + rawRelease: renderTestAutorelease, newName: "other", reject: true, + }, {name: "version downgrade", existing: "2-1", rendered: "1-99", reject: true}, {name: "epoch increase", existing: "9-9", rendered: "1:1-1"}, {name: "epoch decrease", existing: "1:1-1", rendered: "9-9", reject: true}, @@ -67,6 +82,14 @@ func TestLocklessRenderNEVRBeforeReplacement(t *testing.T) { rendered, err := rpm.NewVersion(testCase.rendered) require.NoError(t, err) + rawRelease := rendered.Release() + if testCase.rawRelease != "" { + rawRelease = testCase.rawRelease + } + + require.NoError(t, fileutils.WriteFile(env.TestFS, "/staging/curl/curl.spec", + []byte(renderTestSpec(rendered.Version(), rawRelease, "")), fileperms.PublicFile)) + newName := "curl" if testCase.newName != "" { newName = testCase.newName @@ -108,6 +131,10 @@ func TestLocklessRenderNEVRBeforeReplacement(t *testing.T) { assert.False(t, prep.outputTouched) if !testCase.queryFails && !testCase.toolMissing { + if testCase.rawRelease != "" { + require.ErrorContains(t, err, "new NEVR must be equal to or higher than existing NEVR") + } + require.ErrorContains(t, err, "existing `curl-"+testCase.existing+"`") require.ErrorContains(t, err, "new `"+newName+"-"+testCase.rendered+"`") } @@ -140,18 +167,26 @@ func TestLocklessRenderNEVRBeforeReplacement(t *testing.T) { func TestRenderNEVRSkipsFirstAndLockedRenders(t *testing.T) { for _, existing := range []bool{false, true} { - t.Run(strconv.FormatBool(existing), func(t *testing.T) { - env := testutils.NewTestEnv(t) - setupRenderOutput(t, env.TestFS, existing) - env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { - t.Fatalf("must not query NEVR for first or locked output: %v", cmd.Args) + for _, autorelease := range []bool{false, true} { + t.Run(strconv.FormatBool(existing)+"/"+strconv.FormatBool(autorelease), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, existing) - return "", nil - } - require.NoError(t, moveRenderedOutput(env.Env, "/staging/curl", &preparedComponent{ - lockless: !existing, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, - }, true)) - }) + if autorelease { + require.NoError(t, fileutils.WriteFile(env.TestFS, "/staging/curl/curl.spec", + []byte(renderTestSpec("1", renderTestAutorelease, "")), fileperms.PublicFile)) + } + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("must not query NEVR for first or locked output: %v", cmd.Args) + + return "", nil + } + require.NoError(t, moveRenderedOutput(env.Env, "/staging/curl", &preparedComponent{ + lockless: !existing, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, + }, true)) + }) + } } } From 9c4e37e4398c90338574a0791dfc8df9e4cf0920 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sat, 7 Nov 2026 02:08:17 +0000 Subject: [PATCH 45/70] fix(component): use committed dist-git for lockless builds and queries Enable lockless component queries from temporary filtered clones of the committed rendered dist-git history, resolving rpmautospec metadata before querying. Keep locked-component working-tree queries unchanged. Reject lockless prepare-sources with guidance to use dist-git-repo. Build lockless SRPMs from extracted dist-git using Mock's rpmautospec plugin, then use the existing binary RPM, publication, and no-check paths. Reject incompatible build flags and report missing extraction/plugin dependencies explicitly. Preserve locked-component build behavior. Update user documentation, emitted agent guidance, and focused tests. Validation: mage fix all, mage unit, mage check all, mage docs, and native Git/filter-repo/rpmautospec/CLI checks passed. mage all passed units and quality checks, but scenarios could not create the host Docker provider ($XDG_RUNTIME_DIR). Real Mock builds remain unverified because the host does not have the rpmautospec plugin installed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/build-component.md | 8 + docs/user/how-to/preview-without-lockfile.md | 41 ++++- .../reference/cli/azldev_component_build.md | 6 + .../cli/azldev_component_prepare-sources.md | 2 + .../reference/cli/azldev_component_query.md | 11 +- docs/user/reference/config/components.md | 12 +- .../app/azldev/agentskill/agentskill_test.go | 16 ++ .../azldev/agentskill/content/azldev.md.tmpl | 6 + .../content/build-component.md.tmpl | 8 + .../agentskill/content/comp-toml.md.tmpl | 6 +- .../content/remove-component.md.tmpl | 5 +- .../content/withoutlockfile/azldev.md.tmpl | 6 + .../withoutlockfile/build-component.md.tmpl | 8 + internal/app/azldev/cmds/component/build.go | 10 ++ .../azldev/cmds/component/build_lockless.go | 117 +++++++++++++ .../component/build_lockless_internal_test.go | 133 +++++++++++++++ .../azldev/cmds/component/component_test.go | 6 +- .../azldev/cmds/component/distgitrepo_test.go | 24 +-- .../azldev/cmds/component/preparesources.go | 9 +- internal/app/azldev/cmds/component/query.go | 68 +++++--- .../azldev/cmds/component/query_lockless.go | 105 ++++++++++++ .../app/azldev/cmds/component/query_test.go | 156 ++++++++++++++++++ .../core/componentbuilder/componentbuilder.go | 21 +++ .../componentbuilder/componentbuilder_test.go | 38 +++++ .../app/azldev/core/sources/mockprocessor.go | 2 + .../app/azldev/core/sources/render_process.py | 8 +- internal/rpm/mock/mock.go | 36 +++- internal/rpm/mock/mock_test.go | 37 +++++ 28 files changed, 845 insertions(+), 60 deletions(-) create mode 100644 internal/app/azldev/cmds/component/build_lockless.go create mode 100644 internal/app/azldev/cmds/component/build_lockless_internal_test.go create mode 100644 internal/app/azldev/cmds/component/query_lockless.go diff --git a/docs/user/how-to/build-component.md b/docs/user/how-to/build-component.md index 8cb5701ef..e5b4a5d14 100644 --- a/docs/user/how-to/build-component.md +++ b/docs/user/how-to/build-component.md @@ -26,6 +26,14 @@ Built packages are placed in structured subdirectories under the project's outpu ## Common Options +For components listed in `lockless-components.txt`, render and commit first. +Build extracts the committed component dist-git history using `git-filter-repo`, +downloads lookaside sources, and builds the SRPM with Mock's `rpmautospec` plugin. +Install `git-filter-repo` and Mock 5.3 or newer. Uncommitted rendered edits are +ignored. Lockless builds reject `--without-git` and `--rpmdev-bumpspec`; +lock-file-based builds are unchanged. See the +[lockless workflow](preview-without-lockfile.md#preparing-querying-and-building-committed-dist-git). + | Flag | Description | |------|-------------| | `--no-check` | Skip the `%check` section (package test suite) | diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index c8f41b791..9874b714e 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -60,8 +60,9 @@ Update (including `--bump`) skips listed components and leaves their locks untouched; refresh skips unlisted components and preserves their generated TOML files. All-component scans still prune files for components that no longer exist, not files belonging to components using the other mode. -`component history` and `component query` skip listed components, matching the -existing lockless no-op behavior. `component changed` compares resolved project +`component history` skips listed components. `component query` extracts their +committed rendered dist-git history and processes it with rpmautospec before +querying; uncommitted rendered changes are ignored. `component changed` compares resolved project inputs for listed components and stored fingerprints for unlisted components; the current checkout's list determines which comparison to use. @@ -71,12 +72,44 @@ the current checkout's list determines which comparison to use. |------|---------|----------------------| | Resolved commit storage | `locks/.lock` | `base/upstream-commits/.toml` | | Refresh command | `azldev component update` | `azldev component refresh-upstream-commit` | -| Inspecting resolved state | `azldev component history`, `azldev component query` | read the generated TOML; no equivalent commands | +| Inspecting resolved state | `azldev component history`, `azldev component query` | `component query` inspects committed rendered specs; `history` skips lockless components | | Lock consistency checks | On, with `--skip-lock-validation` to opt out | Skipped | | `component changed` | Compares stored input fingerprints | Compares project configuration resolved at each ref | -| Synthetic dist-git history | Derived from lock-file fingerprint changes | Skipped during rendering; other workflows derive it from generated upstream-commit TOML changes | +| Synthetic dist-git history | Derived from lock-file fingerprint changes | Not generated; build and query extract committed rendered history | | Agent skills and MCP tools | Both workflows are available | Both workflows are available | +### Preparing, querying, and building committed dist-git + +Render and commit the component before querying or building it. Lockless +`component query` and `component build` use the same extraction as +`component dist-git-repo`: a temporary `--single-branch --no-local` clone of the +current project branch, filtered with `git filter-repo --subdirectory-filter` +using the component's repository-relative rendered directory. Install +`git-filter-repo` on the host. Uncommitted rendered changes are not included, +and extraction never modifies the original repository. + +`component query` processes the temporary spec with rpmautospec in the project's +Mock chroot, then reports its source NEVR, required files, and binary subpackages. +The temporary query repositories are removed on success and failure. +Locked queries continue to read working-tree rendered files, including uncommitted edits. + +`component build` downloads sources listed in the extracted `sources` manifest, +then builds the SRPM with Mock's `rpmautospec` plugin enabled and the extracted +repository passed as `--sources`. This requires Mock 5.3 or newer and +rpmautospec available from the build distro's package repositories. +Mock's rpmautospec plugin must be installed on the host as well; enabling an +unavailable plugin is an error, even if Mock itself would only warn. +The plugin resolves `%autorelease` and `%autochangelog` using the filtered +history; no synthetic history, overlay replay, or separate Release bump runs. +Binary RPMs are built from that SRPM normally. `--no-check` is forwarded through +Mock's `--nocheck`; `--srpm-only` skips binary builds. Lockless builds reject +`--without-git` and `--rpmdev-bumpspec`. Locked build behavior is unchanged. + +`component prepare-sources` rejects lockless components before touching its +output directory. Use `azldev component dist-git-repo -o ` +instead to obtain a committed working dist-git repository. This extraction +command does not download source archives. + ## Configure the Project Include the generated directory **before** the component-specific TOML, so that a diff --git a/docs/user/reference/cli/azldev_component_build.md b/docs/user/reference/cli/azldev_component_build.md index 8392d9814..4a12dd2de 100644 --- a/docs/user/reference/cli/azldev_component_build.md +++ b/docs/user/reference/cli/azldev_component_build.md @@ -16,6 +16,12 @@ are placed in structured subdirectories under the project output directory: out/rpms/ - binary RPMs with no channel configured (or channel="none") out/rpms// - binary RPMs moved to their configured publish channel subdirectory +For lockless components, build instead extracts committed rendered dist-git +history using git-filter-repo, downloads lookaside sources, and builds the SRPM +with mock's rpmautospec plugin (mock 5.3+). Uncommitted rendered changes are +ignored. '--without-git' and '--rpmdev-bumpspec' are unsupported for lockless +components. '--no-check' skips the binary build's package checks normally. + The publish channel for each package is resolved from the project's package configuration (package groups and per-component overrides). See 'azldev package' for details. diff --git a/docs/user/reference/cli/azldev_component_prepare-sources.md b/docs/user/reference/cli/azldev_component_prepare-sources.md index e17685d77..59912f1a7 100644 --- a/docs/user/reference/cli/azldev_component_prepare-sources.md +++ b/docs/user/reference/cli/azldev_component_prepare-sources.md @@ -14,6 +14,8 @@ for inspection or manual building. This is useful for verifying that overlays apply cleanly before running a full build. Only one component may be selected at a time. +Lockless components are unsupported; use 'azldev component dist-git-repo' instead +to extract their committed rendered files and history. ``` azldev component prepare-sources [flags] diff --git a/docs/user/reference/cli/azldev_component_query.md b/docs/user/reference/cli/azldev_component_query.md index 12c3ca12c..b90411f47 100644 --- a/docs/user/reference/cli/azldev_component_query.md +++ b/docs/user/reference/cli/azldev_component_query.md @@ -10,13 +10,16 @@ Query detailed information for components from their locally rendered specs. This command reads the post-overlay specs from the project's rendered-specs-dir (produced by 'azldev component render') and runs rpmspec against them in a -single shared mock chroot, batching all specs into one chroot invocation with -parallel per-spec processing. For each component, it reports the source NEVR +single shared mock chroot, querying specs in batches with parallel per-spec +processing. For each component, it reports the source NEVR and the list of binary subpackages the spec would produce when built. -Components using the lockless workflow via lockless-components.txt are skipped. +For lockless components, queries use committed rendered files from a temporary +dist-git repository extracted with git-filter-repo. rpmautospec resolves the +release and changelog in that temporary repository before querying. Uncommitted +rendered changes are ignored. Requires git-filter-repo for lockless components. -The rendered-specs-dir must exist on disk; if it doesn't, run +For locked components, rendered-specs-dir must exist on disk; if it doesn't, run 'azldev component render' first. Components that previously failed to render (those with a RENDER_FAILED marker file) are skipped with a warning. diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index aa3b28ecc..70abc53b8 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -140,7 +140,7 @@ calculation = "manual" ### Optional `rpmdev-bumpspec` preview -By default, build, render, and `prepare-sources` use the legacy static Release +For lock-file-based components, build, render, and `prepare-sources` use the legacy static Release calculation above. It supports only a bare integer or an integer followed by `%{?dist}` or `%{dist}`. Other static forms, such as dotted releases or custom macros, retain the existing non-standard Release error and require `manual` or @@ -153,12 +153,16 @@ uses the existing fixed changelog metadata and requires `rpmdev-bumpspec`, `rpmdev-packager`, `rpm`, `rpmspec`, and Python with the RPM module only when explicitly selected for this transactional strategy. `%autorelease` behavior is unchanged. -**Exception:** Rendering a lockless upstream component always invokes host -`rpmdev-bumpspec -c` before the existing processing, without requiring the flag. -Its `%autorelease` no-op is accepted. See the +**Exception:** Rendering a lockless upstream component with a non-autorelease +Release invokes host `rpmdev-bumpspec -c` without requiring the flag. +Autorelease renders skip bumpspec. See the [lockless rendering workflow](../../how-to/preview-without-lockfile.md#render-and-commit-an-upstream-component) for release/changelog carryover and automatic output commits. +Lockless builds instead use committed rendered dist-git history and Mock's +rpmautospec plugin; they reject `--rpmdev-bumpspec` and `--without-git`. +Lockless `prepare-sources` is unsupported; use `component dist-git-repo` instead. + ## Render Configuration The `[components..render]` section controls rendering behavior for a component. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 5fdca8cbf..4d251762f 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -201,6 +201,22 @@ func TestSkillsDocumentLocklessRPMContext(t *testing.T) { } } +func TestSkillsDocumentLocklessBuildAndQuery(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + catalog := agentskill.NewCatalog(withoutLockfile) + doc, err := catalog.SkillDocument("azldev", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "`comp query` extracts committed rendered history") + assert.Contains(t, doc, "`comp prepare-sources` rejects lockless components") + assert.Contains(t, doc, "locked queries still read the") + doc, err = catalog.SkillDocument("azldev-build-component", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "Mock's rpmautospec plugin (Mock 5.3+)") + assert.Contains(t, doc, "Uncommitted rendered edits are ignored") + assert.Contains(t, doc, "Locked builds retain their existing") + } +} + func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { for _, withoutLockfile := range []bool{false, true} { doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev", testParams()) diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 9a11c3b0d..31305d40c 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -139,6 +139,12 @@ and git-filter-repo. It clones the local project's current branch with Only committed rendered files/history are exported; it does not fetch upstream. Existing output requires `--force`, which deletes it after preflight checks. +For lockless components, `comp query` extracts committed rendered history into +temporary dist-git repositories and processes it with rpmautospec in Mock before +querying. Uncommitted rendered edits are ignored; locked queries still read the +working tree. `comp prepare-sources` rejects lockless components; use +`comp dist-git-repo` instead. `comp history` continues to skip lockless components. + The hidden `advanced` group (`adv`) contains specialist integrations such as MCP and mock helpers; it is intentionally omitted from normal help. diff --git a/internal/app/azldev/agentskill/content/build-component.md.tmpl b/internal/app/azldev/agentskill/content/build-component.md.tmpl index 8f917a47b..0a29fb294 100644 --- a/internal/app/azldev/agentskill/content/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/build-component.md.tmpl @@ -23,6 +23,14 @@ progress — use it only for inner-loop builds you expect to succeed. ## The inner loop +For lockless components, build uses committed rendered dist-git history extracted +with git-filter-repo; render and commit first. It downloads lookaside sources and +builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. +Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` +are rejected for lockless builds; `--no-check` is forwarded normally. +Lockless `comp prepare-sources` is unsupported; use `comp dist-git-repo` instead. +Locked builds retain their existing source preparation and flag behavior. + investigate → modify → render → build → test → inspect | Step | Command | diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index ec6e5fb2d..470d448da 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -86,13 +86,15 @@ calculation = "manual" ``` The temporary `--rpmdev-bumpspec` flag on `comp build`, `comp render`, and -`comp prepare-sources` opts into transactional host handling while Azure Linux +`comp prepare-sources` opts into transactional host handling for locked components while Azure Linux migrates. Only that path requires `rpmdev-bumpspec`, `rpmdev-packager`, `rpm`, `rpmspec`, and Python with its RPM module. It preserves the fixed team identity/comment/datestamp and strict source-EVR transaction checks; locks and synthetic history still determine the number and order of bumps. Without the flag, no bumpspec runtime discovery occurs and the legacy static calculation is -used outside the lockless upstream render workflow. That workflow always runs +used for locked components. Lockless builds reject this flag and use Mock's +rpmautospec plugin with committed dist-git history; lockless `comp prepare-sources` +is unsupported. The lockless upstream render workflow always runs host `rpmdev-bumpspec -c` only for non-autorelease specs and automatically commits the component output. Missing `--changelog-comment` opens `$EDITOR` or a local editor before output replacement, including for autorelease commits. diff --git a/internal/app/azldev/agentskill/content/remove-component.md.tmpl b/internal/app/azldev/agentskill/content/remove-component.md.tmpl index f1d1c902a..369ecb948 100644 --- a/internal/app/azldev/agentskill/content/remove-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/remove-component.md.tmpl @@ -23,8 +23,9 @@ Check for **reverse dependencies** first: if other components `BuildRequires` or `Requires` this one, removing it breaks their builds. Search the tree for the component name. Identify its binary subpackage names from the rendered output under `{{ .RenderedSpecsDir }}///`, then search for those names too. -`azldev comp query` currently supports local spec sources only, not the usual upstream -spec source. +`azldev comp query` reads rendered specs for either source type. Locked components +use working-tree rendered files; lockless components use committed rendered +dist-git history processed with rpmautospec. ## Steps diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index aa4ef1ce5..a9f1a31ac 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -138,6 +138,12 @@ and git-filter-repo. It clones the local project's current branch with Only committed rendered files/history are exported; it does not fetch upstream. Existing output requires `--force`, which deletes it after preflight checks. +For lockless components, `comp query` extracts committed rendered history into +temporary dist-git repositories and processes it with rpmautospec in Mock before +querying. Uncommitted rendered edits are ignored; locked queries still read the +working tree. `comp prepare-sources` rejects lockless components; use +`comp dist-git-repo` instead. `comp history` continues to skip lockless components. + The hidden `advanced` group (`adv`) contains specialist integrations such as MCP and mock helpers; it is intentionally omitted from normal help. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl index 2aa731b31..255ac6abd 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl @@ -23,6 +23,14 @@ progress — use it only for inner-loop builds you expect to succeed. ## The inner loop +For lockless components, build uses committed rendered dist-git history extracted +with git-filter-repo; render and commit first. It downloads lookaside sources and +builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. +Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` +are rejected for lockless builds; `--no-check` is forwarded normally. +Lockless `comp prepare-sources` is unsupported; use `comp dist-git-repo` instead. +Locked builds retain their existing source preparation and flag behavior. + investigate → modify → render → build → test → inspect | Step | Command | diff --git a/internal/app/azldev/cmds/component/build.go b/internal/app/azldev/cmds/component/build.go index 8c879ae9e..a2229d197 100644 --- a/internal/app/azldev/cmds/component/build.go +++ b/internal/app/azldev/cmds/component/build.go @@ -105,6 +105,12 @@ are placed in structured subdirectories under the project output directory: out/rpms/ - binary RPMs with no channel configured (or channel="none") out/rpms// - binary RPMs moved to their configured publish channel subdirectory +For lockless components, build instead extracts committed rendered dist-git +history using git-filter-repo, downloads lookaside sources, and builds the SRPM +with mock's rpmautospec plugin (mock 5.3+). Uncommitted rendered changes are +ignored. '--without-git' and '--rpmdev-bumpspec' are unsupported for lockless +components. '--no-check' skips the binary build's package checks normally. + The publish channel for each package is resolved from the project's package configuration (package groups and per-component overrides). See 'azldev package' for details. @@ -242,6 +248,10 @@ func buildComponent( mockProcessor *sources.MockProcessor, options *ComponentBuildOptions, ) (results ComponentBuildResults, err error) { + if env.ComponentWithoutLockfile(component.GetName()) { + return buildLocklessComponent(env, component, workDirFactory, options) + } + // Resolve the effective distro for this component before creating the source manager. distro, err := sourceproviders.ResolveDistro(env, component) if err != nil { diff --git a/internal/app/azldev/cmds/component/build_lockless.go b/internal/app/azldev/cmds/component/build_lockless.go new file mode 100644 index 000000000..52ca74f61 --- /dev/null +++ b/internal/app/azldev/cmds/component/build_lockless.go @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "fmt" + "path/filepath" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/downloadsources" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/componentbuilder" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/workdir" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/defers" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +func buildLocklessComponent( + env *azldev.Env, component components.Component, factory *workdir.Factory, options *ComponentBuildOptions, +) (results ComponentBuildResults, err error) { + name := component.GetName() + if options.WithoutGitRepo || options.RPMDevBumpspec { + return results, fmt.Errorf("lockless component %#q does not support '--without-git' or '--rpmdev-bumpspec'", name) + } + + dir, err := factory.Create(name, "dist-git") + if err != nil { + return results, fmt.Errorf("creating dist-git work directory:\n%w", err) + } + defer fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), dir, &err) + + if _, err := CreateComponentDistGitRepo(env, &DistGitRepoOptions{ + Component: name, OutputDir: dir, Force: true, + }); err != nil { + return results, err + } + + if err := downloadDistGitSources(env, name, dir); err != nil { + return results, err + } + + buildEnv, err := workdir.MkComponentBuildEnvironment(env, factory, component.GetConfig(), "build", + options.MockConfigOpts) + if err != nil { + return results, fmt.Errorf("creating lockless build environment:\n%w", err) + } + defer defers.HandleDeferError(func() error { + if !options.BuildEnvPolicy.ShouldPreserve(err == nil) { + return buildEnv.Destroy(env) + } + + return nil + }, &err) + + builder := componentbuilder.New(env, env.FS(), env, nil, buildEnv, factory) + cfg := component.GetConfig() + builder.UseDistGitSources(dir, mock.CommonBuildOptions{ + With: cfg.Build.With, Without: cfg.Build.Without, Defines: cfg.Build.Defines, + }) + + repos := append([]string{}, options.LocalRepoPaths...) + if options.LocalRepoWithPublishPath != "" { + repos = append(repos, options.LocalRepoWithPublishPath) + } + + return buildComponentUsingBuilder(env, component, builder, repos, options.SourcePackageOnly, + options.NoCheck, options.LocalRepoWithPublishPath) +} + +func downloadDistGitSources(env *azldev.Env, name, dir string) error { + path := filepath.Join(dir, "sources") + + exists, err := fileutils.Exists(env.FS(), path) + if err != nil { + return fmt.Errorf("checking dist-git 'sources' manifest:\n%w", err) + } + + if !exists { + return nil + } + + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return fmt.Errorf("reading dist-git 'sources' manifest:\n%w", err) + } + + lines, err := fedorasource.ReadSourcesFile(string(content)) + if err != nil { + return fmt.Errorf("parsing dist-git 'sources' manifest:\n%w", err) + } + + for _, line := range lines { + if line.Entry == nil { + continue + } + + exists, err := fileutils.Exists(env.FS(), filepath.Join(dir, line.Entry.Filename)) + if err != nil { + return fmt.Errorf("checking dist-git source %#q:\n%w", line.Entry.Filename, err) + } + + if !exists { + if err := downloadsources.DownloadSources(env, &downloadsources.DownloadSourcesOptions{ + Directory: dir, ComponentName: name, + }); err != nil { + return fmt.Errorf("downloading dist-git sources:\n%w", err) + } + + break + } + } + + return nil +} diff --git a/internal/app/azldev/cmds/component/build_lockless_internal_test.go b/internal/app/azldev/cmds/component/build_lockless_internal_test.go new file mode 100644 index 000000000..8f71d2d9b --- /dev/null +++ b/internal/app/azldev/cmds/component/build_lockless_internal_test.go @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/workdir" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/require" +) + +func TestLocklessBuildRejectsSyntheticHistoryFlags(t *testing.T) { + for _, options := range []ComponentBuildOptions{{WithoutGitRepo: true}, {RPMDevBumpspec: true}} { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + env.Config.Components["curl"] = projectconfig.ComponentConfig{Name: "curl"} + comp := resolveComponents(t, env, "curl")[0] + _, err := BuildComponent(env.Env, comp, nil, &options) + require.ErrorContains(t, err, "does not support '--without-git' or '--rpmdev-bumpspec'") + } +} + +func TestLocklessBuildUsesDistGitAndCleansUp(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(map[bool]string{false: "success", true: "build failure"}[fail], func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.Project.RenderedSpecsDir = "/project/specs" + env.Config.LocklessComponents = map[string]bool{"curl": true} + env.Config.Components["curl"] = projectconfig.ComponentConfig{Name: "curl"} + env.CmdFactory.RegisterCommandInSearchPath("git") + env.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + + var cloneDir string + + sentinel := errors.New("injected SRPM failure") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[0] != "git" { + return "/mock/root", nil + } + + switch cmd.Args[3] { + case "rev-parse": + if cmd.Args[4] == "--git-common-dir" { + return filepath.Join("/project", ".git"), nil + } + + return "/project", nil + case "clone": + cloneDir = cmd.Args[len(cmd.Args)-1] + require.Equal(t, []string{"--single-branch", "--no-local", "--"}, cmd.Args[4:7]) + require.NoError(t, fileutils.MkdirAll(env.FS(), cloneDir)) + case "filter-repo": + if slices.Contains(cmd.Args, "--subdirectory-filter") { + require.Equal(t, "specs/c/curl", cmd.Args[len(cmd.Args)-1]) + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(cloneDir, "curl.spec"), + []byte("committed spec"), fileperms.PublicFile)) + } + } + + return "", nil + } + built := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + require.NotContains(t, strings.Join(cmd.Args, " "), "calculate-release") + + if !slices.Contains(cmd.Args, "--buildsrpm") { + return nil + } + + built = true + + require.Contains(t, strings.Join(cmd.Args, " "), "--enable-plugin rpmautospec") + require.Equal(t, cloneDir, cmd.Args[slices.Index(cmd.Args, "--sources")+1]) + + if fail { + return sentinel + } + + out := cmd.Args[slices.Index(cmd.Args, "--resultdir")+1] + + return fileutils.WriteFile(env.FS(), filepath.Join(out, "curl.src.rpm"), []byte("srpm"), fileperms.PublicFile) + } + factory, err := workdir.NewFactory(env.FS(), env.Env.WorkDir(), env.Env.ConstructionTime()) + require.NoError(t, err) + comp := resolveComponents(t, env, "curl")[0] + + results, err := BuildComponent(env.Env, comp, factory, &ComponentBuildOptions{SourcePackageOnly: true}) + if fail { + require.ErrorIs(t, err, sentinel) + } else { + require.NoError(t, err) + require.Len(t, results.SRPMPaths, 1) + } + + require.True(t, built) + + exists, err := fileutils.Exists(env.FS(), cloneDir) + require.NoError(t, err) + require.False(t, exists) + }) + } +} + +func TestLocklessPrepareSourcesPreservesOutput(t *testing.T) { + for _, options := range []PrepareSourcesOptions{ + {}, {Force: true}, {SkipOverlays: true}, {WithoutGitRepo: true}, {RPMDevBumpspec: true}, + } { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + env.Config.Components["curl"] = projectconfig.ComponentConfig{Name: "curl"} + options.ComponentFilter = components.ComponentFilter{ComponentNamePatterns: []string{"curl"}} + options.OutputDir = "/output" + + require.NoError(t, fileutils.WriteFile(env.FS(), "/output/keep", []byte("original"), fileperms.PublicFile)) + err := PrepareComponentSources(env.Env, &options) + require.ErrorContains(t, err, "use 'azldev component dist-git-repo' instead") + content, err := fileutils.ReadFile(env.FS(), "/output/keep") + require.NoError(t, err) + require.Equal(t, "original", string(content)) + } +} diff --git a/internal/app/azldev/cmds/component/component_test.go b/internal/app/azldev/cmds/component/component_test.go index 907d702ee..73e77ab40 100644 --- a/internal/app/azldev/cmds/component/component_test.go +++ b/internal/app/azldev/cmds/component/component_test.go @@ -52,7 +52,7 @@ func TestNewComponentCommands_LockValidationFlags(t *testing.T) { assert.NotNil(t, component.NewUpdateCmd().Flags().Lookup("skip-lock-validation")) } -func TestLocklessComponents_SkipLegacyHistoryAndQuery(t *testing.T) { +func TestLocklessComponents_SkipLegacyHistory(t *testing.T) { env := testutils.NewTestEnv(t) env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} env.Config.Components["rust-asyncband"] = projectconfig.ComponentConfig{ @@ -67,8 +67,4 @@ func TestLocklessComponents_SkipLegacyHistoryAndQuery(t *testing.T) { }) require.NoError(t, err) assert.Empty(t, history) - - query, err := component.QueryComponents(env.Env, &component.QueryComponentsOptions{ComponentFilter: filter}) - require.NoError(t, err) - assert.Empty(t, query) } diff --git a/internal/app/azldev/cmds/component/distgitrepo_test.go b/internal/app/azldev/cmds/component/distgitrepo_test.go index 343798065..ffb3212cb 100644 --- a/internal/app/azldev/cmds/component/distgitrepo_test.go +++ b/internal/app/azldev/cmds/component/distgitrepo_test.go @@ -20,11 +20,13 @@ import ( ) const ( - distGitTestOutput = "/exports/component" - distGitTestProject = "/project" - distGitTestRevParse = "rev-parse" - distGitTestCommonDir = "--git-common-dir" - distGitTestClone = "clone" + distGitTestOutput = "/exports/component" + distGitTestProject = "/project" + distGitTestRevParse = "rev-parse" + distGitTestCommonDir = "--git-common-dir" + distGitTestClone = "clone" + distGitTestFilterRepo = "filter-repo" + distGitTestMetadata = "/project/.git" ) func newDistGitRepoTestEnv(t *testing.T, name string) *testutils.TestEnv { @@ -88,7 +90,7 @@ func TestCreateComponentDistGitRepo(t *testing.T) { switch cmd.Args[3] { case distGitTestRevParse: if cmd.Args[4] == distGitTestCommonDir { - return "/project/.git", nil + return distGitTestMetadata, nil } return distGitTestProject, nil @@ -108,14 +110,14 @@ func TestCreateComponentDistGitRepo(t *testing.T) { assert.Equal(t, testCase.component, result.Component) assert.Equal(t, distGitTestOutput, result.OutputDir) assert.Equal(t, [][]string{ - {"git", "-C", distGitTestProject, "filter-repo", "--version"}, + {"git", "-C", distGitTestProject, distGitTestFilterRepo, "--version"}, {"git", "-C", distGitTestProject, distGitTestRevParse, "--show-toplevel"}, {"git", "-C", distGitTestProject, distGitTestRevParse, distGitTestCommonDir}, { "git", "-C", distGitTestProject, distGitTestClone, "--single-branch", "--no-local", "--", distGitTestProject, distGitTestOutput, }, - {"git", "-C", distGitTestOutput, "filter-repo", "--subdirectory-filter", testCase.wantSubdir}, + {"git", "-C", distGitTestOutput, distGitTestFilterRepo, "--subdirectory-filter", testCase.wantSubdir}, }, commands) }) } @@ -150,7 +152,7 @@ func TestCreateComponentDistGitRepo_ExistingOutput(t *testing.T) { } func TestCreateComponentDistGitRepo_FailuresPreserveOutput(t *testing.T) { - for _, step := range []string{"filter-repo", distGitTestRevParse} { + for _, step := range []string{distGitTestFilterRepo, distGitTestRevParse} { t.Run(step, func(t *testing.T) { env := newDistGitRepoTestEnv(t, "curl") require.NoError(t, fileutils.MkdirAll(env.TestFS, distGitTestOutput)) @@ -173,7 +175,7 @@ func TestCreateComponentDistGitRepo_FailuresPreserveOutput(t *testing.T) { }) require.ErrorIs(t, err, failure) - if step == "filter-repo" { + if step == distGitTestFilterRepo { require.ErrorContains(t, err, "git-filter-repo must be installed to use the dist-git-repo command") } @@ -237,7 +239,7 @@ func TestCreateComponentDistGitRepo_CommandFailures(t *testing.T) { } cloned = true - case "filter-repo": + case distGitTestFilterRepo: if cmd.Args[4] == "--subdirectory-filter" { assert.True(t, cloned) diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index f234712b5..d2ee899e9 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -49,7 +49,9 @@ The result is a directory containing the spec file and all sources, ready for inspection or manual building. This is useful for verifying that overlays apply cleanly before running a full build. -Only one component may be selected at a time.`, +Only one component may be selected at a time. +Lockless components are unsupported; use 'azldev component dist-git-repo' instead +to extract their committed rendered files and history.`, Example: ` # Prepare sources for a component azldev component prep-sources -p curl -o ./build/work/scratch/curl --force @@ -114,6 +116,11 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er component := comps.Components()[0] + if env.ComponentWithoutLockfile(component.GetName()) { + return fmt.Errorf("component %#q uses the lockless workflow; use 'azldev component dist-git-repo' instead", + component.GetName()) + } + event := env.StartEvent("Preparing sources", "component", component.GetName(), "outputDir", options.OutputDir) defer event.End() diff --git a/internal/app/azldev/cmds/component/query.go b/internal/app/azldev/cmds/component/query.go index b6d097059..7d279b4dd 100644 --- a/internal/app/azldev/cmds/component/query.go +++ b/internal/app/azldev/cmds/component/query.go @@ -8,6 +8,7 @@ import ( "fmt" "log/slog" "path/filepath" + "slices" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" @@ -49,13 +50,16 @@ func NewComponentQueryCommand() *cobra.Command { This command reads the post-overlay specs from the project's rendered-specs-dir (produced by 'azldev component render') and runs rpmspec against them in a -single shared mock chroot, batching all specs into one chroot invocation with -parallel per-spec processing. For each component, it reports the source NEVR +single shared mock chroot, querying specs in batches with parallel per-spec +processing. For each component, it reports the source NEVR and the list of binary subpackages the spec would produce when built. -Components using the lockless workflow via lockless-components.txt are skipped. +For lockless components, queries use committed rendered files from a temporary +dist-git repository extracted with git-filter-repo. rpmautospec resolves the +release and changelog in that temporary repository before querying. Uncommitted +rendered changes are ignored. Requires git-filter-repo for lockless components. -The rendered-specs-dir must exist on disk; if it doesn't, run +For locked components, rendered-specs-dir must exist on disk; if it doesn't, run 'azldev component render' first. Components that previously failed to render (those with a RENDER_FAILED marker file) are skipped with a warning.`, Example: ` # Query a single component @@ -115,31 +119,17 @@ func QueryComponents( resolver := components.NewResolver(env) comps, resolveErr := resolver.FindComponents(&options.ComponentFilter) - // Preserve directory diagnostics unless the selection is entirely lockless. - var lockedComps []components.Component + var lockedComps, locklessComps []components.Component + if resolveErr == nil { - lockedComps = componentsForLockMode(env, comps.Components(), false) - if comps.Len() > 0 && len(lockedComps) == 0 { - return []*componentDetails{}, nil - } + lockedComps, locklessComps = splitQueryComponents(env, comps.Components()) } renderedSpecsDir := env.Config().Project.RenderedSpecsDir - if renderedSpecsDir == "" { - return nil, errors.New( - "project.rendered-specs-dir is not configured; " + - "set it in the project config and run 'azldev component render' first") - } - dirExists, err := fileutils.DirExists(env.FS(), renderedSpecsDir) - if err != nil { - return nil, fmt.Errorf("checking rendered-specs-dir %#q:\n%w", renderedSpecsDir, err) - } - - if !dirExists { - return nil, fmt.Errorf( - "rendered-specs-dir %#q does not exist; run 'azldev component render' first", - renderedSpecsDir) + checkRenderedDir := len(locklessComps) == 0 || len(lockedComps) > 0 + if err := validateQueryRenderedDirectory(env, renderedSpecsDir, checkRenderedDir); err != nil { + return nil, err } if resolveErr != nil { @@ -153,12 +143,18 @@ func QueryComponents( "please use command-line options to indicate which components to query") } + if len(locklessComps) > 0 { + if err := requireGitFilterRepo(env); err != nil { + return nil, err + } + } + inputs, _, err := buildSpecQueryInputs(env, lockedComps, renderedSpecsDir) if err != nil { return nil, err } - if len(inputs) == 0 { + if len(inputs) == 0 && len(locklessComps) == 0 { return nil, fmt.Errorf( "none of the %d resolved components have a rendered spec on disk; "+ "run 'azldev component render' first", @@ -166,6 +162,10 @@ func QueryComponents( } mockProcessor := createQueryMockProcessor(env) + if len(locklessComps) > 0 { + mockProcessor = newMockProcessor(env, "", append(mockPackagesForQuery(), mockPackagesForRender()...)) + } + if mockProcessor == nil { return nil, errors.New( "mock config required for querying; ensure the project has a valid distro with mock config") @@ -201,6 +201,24 @@ func QueryComponents( return nil, fmt.Errorf("batch-querying rendered specs:\n%w", err) } + if len(locklessComps) > 0 { + locklessResults, queryErr := queryLocklessComponents(env, mockProcessor, locklessComps, scratchDir, archStr) + if queryErr != nil { + return nil, queryErr + } + + queryResults = append(queryResults, locklessResults...) + + order := make(map[string]int, comps.Len()) + for idx, comp := range comps.Components() { + order[comp.GetName()] = idx + } + + slices.SortStableFunc(queryResults, func(a, b sources.SpecQueryResult) int { + return order[a.Name] - order[b.Name] + }) + } + allDetails := make([]*componentDetails, 0, len(queryResults)) var ( diff --git a/internal/app/azldev/cmds/component/query_lockless.go b/internal/app/azldev/cmds/component/query_lockless.go new file mode 100644 index 000000000..4546200a6 --- /dev/null +++ b/internal/app/azldev/cmds/component/query_lockless.go @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +func splitQueryComponents( + env *azldev.Env, comps []components.Component, +) (locked, lockless []components.Component) { + for _, comp := range comps { + if env.ComponentWithoutLockfile(comp.GetName()) { + lockless = append(lockless, comp) + } else { + locked = append(locked, comp) + } + } + + return locked, lockless +} + +func validateQueryRenderedDirectory(env *azldev.Env, dir string, required bool) error { + if !required { + return nil + } + + if dir == "" { + return errors.New("project.rendered-specs-dir is not configured; " + + "set it in the project config and run 'azldev component render' first") + } + + exists, err := fileutils.DirExists(env.FS(), dir) + if err != nil { + return fmt.Errorf("checking rendered-specs-dir %#q:\n%w", dir, err) + } + + if !exists { + return fmt.Errorf("rendered-specs-dir %#q does not exist; run 'azldev component render' first", dir) + } + + return nil +} + +func queryLocklessComponents( + env *azldev.Env, processor *sources.MockProcessor, comps []components.Component, scratchDir, arch string, +) ([]sources.SpecQueryResult, error) { + stagingDir := filepath.Join(scratchDir, "dist-git") + processInputs := make([]sources.ComponentInput, 0, len(comps)) + queryInputs := make([]sources.SpecQueryInput, 0, len(comps)) + + for _, comp := range comps { + name := comp.GetName() + if _, err := CreateComponentDistGitRepo(env, &DistGitRepoOptions{ + Component: name, OutputDir: filepath.Join(stagingDir, name), + }); err != nil { + return nil, fmt.Errorf("extracting dist-git for %#q:\n%w", name, err) + } + + specFilename := name + ".spec" + processInputs = append(processInputs, sources.ComponentInput{ + Name: name, SpecFilename: specFilename, SkipSpectool: true, + }) + cfg := comp.GetConfig() + queryInputs = append(queryInputs, sources.SpecQueryInput{ + Name: name, SpecRelPath: filepath.ToSlash(filepath.Join(name, specFilename)), + With: cfg.Build.With, Without: cfg.Build.Without, Defines: cfg.Build.Defines, + }) + } + + processed, err := processor.BatchProcess(env, env, stagingDir, processInputs, env.FS(), env.CPUBoundConcurrency()) + if err != nil { + return nil, fmt.Errorf("processing lockless dist-git specs:\n%w", err) + } + + var ( + failures []sources.SpecQueryResult + ready []sources.SpecQueryInput + ) + + for idx, result := range processed { + if result.Error != nil { + failures = append(failures, sources.SpecQueryResult{Name: result.Name, Error: result.Error}) + } else { + ready = append(ready, queryInputs[idx]) + } + } + + results, err := processor.BatchQuerySpecs(env, env, ready, sources.BatchQuerySpecsOptions{ + SpecsDir: stagingDir, ScratchDir: scratchDir, Arch: arch, FS: env.FS(), MaxWorkers: env.CPUBoundConcurrency(), + }) + if err != nil { + return nil, fmt.Errorf("querying lockless dist-git specs:\n%w", err) + } + + return append(results, failures...), nil +} diff --git a/internal/app/azldev/cmds/component/query_test.go b/internal/app/azldev/cmds/component/query_test.go index 8e265ffbf..eb388ba15 100644 --- a/internal/app/azldev/cmds/component/query_test.go +++ b/internal/app/azldev/cmds/component/query_test.go @@ -5,10 +5,12 @@ package component_test import ( "encoding/json" + "errors" "fmt" "os/exec" "path/filepath" "slices" + "strings" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" @@ -85,6 +87,160 @@ func TestNewComponentQueryCommand(t *testing.T) { assert.Equal(t, "query", cmd.Use) } +func TestQueryComponents_LocklessCommittedDistGit(t *testing.T) { + for _, testCase := range []struct { + name, failure string + mixed bool + }{ + {name: "lockless"}, + {name: "mixed", mixed: true}, + {name: "missing filter-repo", failure: distGitTestFilterRepo}, + {name: "processing failure", failure: "process"}, + {name: "query failure", failure: "query"}, + } { + t.Run(testCase.name, func(t *testing.T) { + failure := testCase.failure + env := newDistGitRepoTestEnv(t, testComponentName) + + env.Config.LocklessComponents = map[string]bool{testComponentName: true} + if testCase.mixed { + env.Config.Components["bash"] = projectconfig.ComponentConfig{ + Name: "bash", + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal, Path: "/project/bash.spec"}, + } + prepareSuccessfulBatchQuery(t, env, "bash", "/project/bash.spec") + } + + lockedQueryHandler := env.CmdFactory.RunHandler + env.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + + sentinel := errors.New("injected failure") + + var ( + cloneDir string + cloned, filtered, processed, queried bool + ) + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[0] != "git" { + return "", nil + } + + switch cmd.Args[3] { + case "rev-parse": + if cmd.Args[4] == "--git-common-dir" { + return distGitTestMetadata, nil + } + + return "/project", nil + case distGitTestClone: + assert.Equal(t, []string{"--single-branch", "--no-local", "--"}, + cmd.Args[4:7]) + cloneDir = cmd.Args[len(cmd.Args)-1] + cloned = true + + require.NoError(t, fileutils.MkdirAll(env.FS(), cloneDir)) + case distGitTestFilterRepo: + if failure == distGitTestFilterRepo { + return "", sentinel + } + + if slices.Contains(cmd.Args, "--subdirectory-filter") { + assert.Equal(t, "specs/c/curl", cmd.Args[len(cmd.Args)-1]) + + filtered = true + + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(cloneDir, "curl.spec"), + []byte("committed spec"), fileperms.PublicFile)) + } + } + + return "", nil + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + args := strings.Join(cmd.Args, " ") + if !strings.Contains(args, "--chroot") { + return nil + } + + if strings.Contains(args, "render_process.py") { + if failure == "process" { + return sentinel + } + + processed = true + + assert.True(t, filtered) + + content, err := fileutils.ReadFile(env.FS(), filepath.Join(cloneDir, "curl.spec")) + require.NoError(t, err) + assert.Equal(t, "committed spec", string(content)) + + return fileutils.WriteFile(env.FS(), filepath.Join(filepath.Dir(cloneDir), "results.json"), + []byte(`[{"name":"curl","specFiles":"","error":null}]`), fileperms.PublicFile) + } + + if strings.Contains(args, "query_process.py") { + if testCase.mixed && !filtered { + return lockedQueryHandler(cmd) + } + + if failure == "query" { + return sentinel + } + + queried = true + + assert.True(t, processed) + + return fileutils.WriteFile(env.FS(), filepath.Join(filepath.Dir(filepath.Dir(cloneDir)), "results.json"), + []byte(`[{"name":"curl","srpmOut":"name=curl\nepoch=0\nversion=1\nrelease=3\n","binOut":"","error":null}]`), + fileperms.PublicFile) + } + + return nil + } + + filter := components.ComponentFilter{ComponentNamePatterns: []string{testComponentName}} + if testCase.mixed { + filter.IncludeAllComponents = true + } + + results, queryErr := component.QueryComponents(env.Env, &component.QueryComponentsOptions{ + ComponentFilter: filter, + }) + dirs, err := fileutils.Glob(env.FS(), filepath.Join(env.Env.WorkDir(), "azldev-query-scratch-*")) + require.NoError(t, err) + assert.Empty(t, dirs, "temporary clones must be removed after success or failure") + + if failure != "" { + require.ErrorIs(t, queryErr, sentinel) + + if failure == distGitTestFilterRepo { + assert.Contains(t, queryErr.Error(), "must be installed") + } + + return + } + + require.NoError(t, queryErr) + + wantLen := 1 + if testCase.mixed { + wantLen = 2 + } + + require.Len(t, results, wantLen) + assert.True(t, cloned && filtered && processed && queried) + + releases := map[string]string{testComponentName: "3", "bash": "1.azl4"} + for _, result := range results { + assert.Equal(t, releases[result.Name], result.Version.Release()) + } + }) + } +} + func TestComponentQueryCmd_NoMatch(t *testing.T) { const testComponentName = "test-component" diff --git a/internal/app/azldev/core/componentbuilder/componentbuilder.go b/internal/app/azldev/core/componentbuilder/componentbuilder.go index 8cb376195..57dbd2fc2 100644 --- a/internal/app/azldev/core/componentbuilder/componentbuilder.go +++ b/internal/app/azldev/core/componentbuilder/componentbuilder.go @@ -46,6 +46,8 @@ type Builder struct { sourcePreparer sources.SourcePreparer buildEnv buildenv.RPMAwareBuildEnv workDirFactory *workdir.Factory + distGitDir string + buildOptions mock.CommonBuildOptions } // Ensure that [Builder] implements the [ComponentBuilder] interface. @@ -80,6 +82,12 @@ func (b *Builder) BuildSourcePackage( return packagePath, nil } +// UseDistGitSources builds the prepared repository without replaying overlays or synthetic history. +func (b *Builder) UseDistGitSources(dir string, options mock.CommonBuildOptions) { + b.distGitDir = dir + b.buildOptions = options +} + func (b *Builder) BuildBinaryPackage( ctx context.Context, component components.Component, sourcePackagePath string, localRepoPaths []string, @@ -114,6 +122,10 @@ func (b *Builder) buildSRPM( func (b *Builder) prepSourcesForSRPM( ctx context.Context, component components.Component, ) (preparedSourcesDir string, err error) { + if b.distGitDir != "" { + return b.distGitDir, nil + } + // Prep sources. prepEvent := b.eventListener.StartEvent("Acquiring and preparing sources", "component", component.GetName()) @@ -162,6 +174,11 @@ func (b *Builder) buildSRPMFromPreparedSources( LocalRepoPaths: localRepoPaths, }, } + if b.distGitDir != "" { + srpmOptions.CommonBuildOptions = b.buildOptions + srpmOptions.LocalRepoPaths = localRepoPaths + srpmOptions.RPMAutospec = true + } err = buildEnv.BuildSRPM(ctx, preparedSpecPath, preparedSourcesDir, srpmOutputDir, srpmOptions) if err != nil { @@ -231,6 +248,10 @@ func (b *Builder) buildRPM( NoCheck: noCheck, ForceRebuild: false, // Could be added to config if needed } + if b.distGitDir != "" { + rpmOptions.CommonBuildOptions = b.buildOptions + rpmOptions.LocalRepoPaths = localRepoPaths + } err = buildEnv.BuildRPM(ctx, srpmPath, tempRPMDir, rpmOptions) if err != nil { diff --git a/internal/app/azldev/core/componentbuilder/componentbuilder_test.go b/internal/app/azldev/core/componentbuilder/componentbuilder_test.go index 7fdf7ec88..8111e6f0e 100644 --- a/internal/app/azldev/core/componentbuilder/componentbuilder_test.go +++ b/internal/app/azldev/core/componentbuilder/componentbuilder_test.go @@ -23,6 +23,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/stretchr/testify/require" @@ -159,6 +160,43 @@ func TestBuilderNew(t *testing.T) { require.NotNil(t, params) } +func TestBuildFromDistGit(t *testing.T) { + params := setupBuilder(t) + comp := params.createTestComponent(t) + + const dir = "/committed-dist-git" + + opts := mock.CommonBuildOptions{With: []string{"feature"}, Defines: map[string]string{"custom": "1"}} + // A nil preparer ensures dist-git builds cannot accidentally replay the old preparation pipeline. + builder := componentbuilder.New(params.testEnv.Env, params.testEnv.FS(), params.testEnv.Env, + nil, params.buildEnv, params.workDirFactory) + builder.UseDistGitSources(dir, opts) + params.buildEnv.EXPECT().BuildSRPM(gomock.Any(), dir+"/"+comp.GetName()+".spec", dir, + gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _, _, output string, options buildenv.SRPMBuildOptions) error { + require.True(t, options.RPMAutospec) + require.Equal(t, opts.With, options.With) + require.Equal(t, opts.Defines, options.Defines) + require.Equal(t, []string{"/local-repo"}, options.LocalRepoPaths) + + return fileutils.WriteFile(params.testEnv.FS(), filepath.Join(output, "package.src.rpm"), + []byte("srpm"), fileperms.PublicFile) + }) + srpm, err := builder.BuildSourcePackage(params.testEnv.Env, comp, []string{"/local-repo"}, "/output") + require.NoError(t, err) + params.buildEnv.EXPECT().BuildRPM(gomock.Any(), srpm, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ string, _ string, options buildenv.RPMBuildOptions) error { + require.True(t, options.NoCheck) + require.Equal(t, opts.With, options.With) + require.Equal(t, opts.Defines, options.Defines) + require.Equal(t, []string{"/local-repo"}, options.LocalRepoPaths) + + return nil + }) + _, err = builder.BuildBinaryPackage(params.testEnv.Env, comp, srpm, []string{"/local-repo"}, "/output", true) + require.NoError(t, err) +} + func TestBuildSourcePackage(t *testing.T) { const testOutputDir = "/output" diff --git a/internal/app/azldev/core/sources/mockprocessor.go b/internal/app/azldev/core/sources/mockprocessor.go index a10ff7107..9fb2c0a72 100644 --- a/internal/app/azldev/core/sources/mockprocessor.go +++ b/internal/app/azldev/core/sources/mockprocessor.go @@ -87,6 +87,7 @@ func NewMockProcessor(ctx opctx.Ctx, mockConfigPath string, opts ...MockProcesso type ComponentInput struct { Name string // Component name (matches subdirectory in staging dir) SpecFilename string // Just the filename, e.g., "curl.spec" + SkipSpectool bool } // ComponentMockResult holds the mock processing result for one component. @@ -402,6 +403,7 @@ func (p *MockProcessor) CalculateRelease(ctx context.Context, specHostDir, specF type componentInputJSON struct { Name string `json:"name"` SpecFilename string `json:"specFilename"` + SkipSpectool bool `json:"skipSpectool,omitempty"` } // componentResultJSON mirrors the JSON output from render_process.py. diff --git a/internal/app/azldev/core/sources/render_process.py b/internal/app/azldev/core/sources/render_process.py index 041a54f61..f8d7f58d1 100755 --- a/internal/app/azldev/core/sources/render_process.py +++ b/internal/app/azldev/core/sources/render_process.py @@ -40,7 +40,9 @@ EXPECTED_ARG_COUNT = 3 -def process_component(staging_dir: str, name: str, spec_filename: str) -> dict[str, str | None]: +def process_component( + staging_dir: str, name: str, spec_filename: str, *, skip_spectool: bool = False +) -> dict[str, str | None]: """Run rpmautospec + spectool for a single component, returning a result dict. Trust boundary: name and spec_filename are validated by BatchProcess in @@ -65,6 +67,9 @@ def process_component(staging_dir: str, name: str, spec_filename: str) -> dict[s "error": f"rpmautospec failed: {rpa_result.stderr.strip()}", } + if skip_spectool: + return {"name": name, "specFiles": "", "error": None} + # spectool: list source and patch files. st_result = subprocess.run( [ @@ -120,6 +125,7 @@ def main() -> int: staging_dir, comp["name"], comp["specFilename"], + skip_spectool=comp.get("skipSpectool", False), ): comp["name"] for comp in inputs } diff --git a/internal/rpm/mock/mock.go b/internal/rpm/mock/mock.go index 566aa4973..88bec8dd9 100644 --- a/internal/rpm/mock/mock.go +++ b/internal/rpm/mock/mock.go @@ -4,6 +4,7 @@ package mock import ( + "bytes" "context" "errors" "fmt" @@ -294,7 +295,7 @@ type CommonBuildOptions struct { // SRPMBuildOptions encapsulates options for building source RPMs using mock. type SRPMBuildOptions struct { CommonBuildOptions - // Add any SRPM-specific options here in the future if needed + RPMAutospec bool } // RPMBuildOptions encapsulates options for building binary RPMs using mock. @@ -324,6 +325,14 @@ func (r *Runner) BuildSRPM( "--resultdir", outputDirPath, ) + if options.RPMAutospec { + if err := r.requireRPMAutospec(ctx); err != nil { + return err + } + + args = append(args, "--enable-plugin", "rpmautospec") + } + // Add 'with'-style enabled conditionals. for _, with := range options.With { args = append(args, "--with", with) @@ -373,6 +382,31 @@ func (r *Runner) BuildSRPM( return nil } +func (r *Runner) requireRPMAutospec(ctx context.Context) error { + args := append(r.getBaseArgs(), "--enable-plugin", "rpmautospec", "--print-root-path") + raw := exec.CommandContext(ctx, MockBinary, args...) + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + cmd, err := r.cmdFactory.Command(raw) + if err != nil { + return fmt.Errorf("checking mock rpmautospec plugin:\n%w", err) + } + + if _, err := cmd.RunAndGetOutput(ctx); err != nil { + return fmt.Errorf("checking mock rpmautospec plugin:\n%w", err) + } + + if strings.Contains(stderr.String(), "rpmautospec plugin") && + strings.Contains(stderr.String(), "not installed") { + return errors.New("mock's rpmautospec plugin must be installed for lockless builds (requires mock 5.3 or newer)") + } + + return nil +} + // Given a path to a Source RPM (SRPM), invokes mock to build a binary RPM. func (r *Runner) BuildRPM(ctx context.Context, srpmPath, outputDirPath string, options RPMBuildOptions) error { slog.Debug("Building RPM", "SRPM", srpmPath) diff --git a/internal/rpm/mock/mock_test.go b/internal/rpm/mock/mock_test.go index 3d402c9ce..0e0231bf8 100644 --- a/internal/rpm/mock/mock_test.go +++ b/internal/rpm/mock/mock_test.go @@ -6,6 +6,7 @@ package mock_test import ( "errors" "fmt" + "io" "os/exec" "path/filepath" "strings" @@ -145,6 +146,25 @@ func TestBuildSRPM(t *testing.T) { assert.Contains(t, mockCmd, "-r "+testMockConfigPath) } +func TestBuildSRPM_MissingRPMAutospecPlugin(t *testing.T) { + ctx := newTestCtxWithMockPrereqsPresent() + ctx.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + _, err := io.WriteString(cmd.Stderr, + "WARNING: rpmautospec plugin is enabled in configuration but is not installed\n") + + return testMockRootPath, err + } + ctx.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + t.Fatal("must not build without the requested rpmautospec plugin") + + return nil + } + runner := mock.NewRunner(ctx, testMockConfigPath) + err := runner.BuildSRPM(ctx, testSpecPath, testSourceDirPath, testOutputDirPath, + mock.SRPMBuildOptions{RPMAutospec: true}) + require.ErrorContains(t, err, "rpmautospec plugin must be installed") +} + func TestBuildSRPM_MockFails(t *testing.T) { testError := errors.New("injected mock failure") @@ -158,6 +178,23 @@ func TestBuildSRPM_MockFails(t *testing.T) { require.ErrorIs(t, err, testError) } +func TestBuildSRPM_RPMAutospec(t *testing.T) { + ctx := newTestCtxWithMockPrereqsPresent() + + var args []string + + ctx.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + args = cmd.Args + + return nil + } + runner := mock.NewRunner(ctx, testMockConfigPath) + require.NoError(t, runner.BuildSRPM(ctx, testSpecPath, testSourceDirPath, testOutputDirPath, + mock.SRPMBuildOptions{RPMAutospec: true})) + assert.Contains(t, strings.Join(args, " "), "--enable-plugin rpmautospec") + assert.Contains(t, strings.Join(args, " "), "--sources "+testSourceDirPath) +} + func TestBuildRPM(t *testing.T) { const ( withoutFeature = "without-feature" From fb6f8a77c07bf38223056c1df51828a0aeb342b6 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sat, 7 Nov 2026 06:13:59 +0000 Subject: [PATCH 46/70] feat(component): support lockless source preparation Prepare lockless components directly in the requested output directory using the locked source acquisition, macro, overlay and hash pipeline. Read release/changelog history from the configured rendered-specs tree without generating synthetic history or committing prepared changes. Honor source-skipping and missing-hash flags, and remove Git metadata only after successful preparation when --without-git is requested. Preserve partial output on failure and use the shared --force output guard. Update command help, generated docs, agent skills and regression tests. Validation: mage fix all, mage unit and mage docs passed. Quality checks in mage all passed; scenario tests and mage scenarioUpdate remain blocked by the existing Docker-provider/$XDG_RUNTIME_DIR initialization failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 33 +- .../cli/azldev_component_prepare-sources.md | 14 +- .../app/azldev/agentskill/agentskill_test.go | 5 +- .../azldev/agentskill/content/azldev.md.tmpl | 17 +- .../content/build-component.md.tmpl | 9 +- .../agentskill/content/comp-toml.md.tmpl | 9 +- .../content/withoutlockfile/azldev.md.tmpl | 17 +- .../withoutlockfile/build-component.md.tmpl | 9 +- .../component/build_lockless_internal_test.go | 20 - .../azldev/cmds/component/preparesources.go | 32 +- .../cmds/component/preparesources_lockless.go | 144 +++++ .../preparesources_lockless_internal_test.go | 554 ++++++++++++++++++ .../cmds/component/preparesources_test.go | 9 +- .../app/azldev/core/sources/sourceprep.go | 2 +- .../azldev/core/sources/sourceprep_test.go | 31 +- 15 files changed, 852 insertions(+), 53 deletions(-) create mode 100644 internal/app/azldev/cmds/component/preparesources_lockless.go create mode 100644 internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 9874b714e..04b66c060 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -105,10 +105,35 @@ Binary RPMs are built from that SRPM normally. `--no-check` is forwarded through Mock's `--nocheck`; `--srpm-only` skips binary builds. Lockless builds reject `--without-git` and `--rpmdev-bumpspec`. Locked build behavior is unchanged. -`component prepare-sources` rejects lockless components before touching its -output directory. Use `azldev component dist-git-repo -o ` -instead to obtain a committed working dist-git repository. This extraction -command does not download source archives. +`component prepare-sources` uses lockless render preparation while writing +directly into `--output-dir`, allowing uncommitted and untracked project files. +Nonempty output requires `--force`, which removes existing contents before +preparation. Failures can leave partially prepared output. It fetches upstream +lookaside payloads with the spec/sidecars, prepares the upstream release/changelog, +then downloads/generates configured `source-files`. Like locked preparation, it +acquires payloads before writing the macro file and applying overlays. Archive +overlays run first, followed by macro-file loading, configured spec/loose-file +overlays, and generated check-skip/header overlays. It updates `sources` metadata +with the configured post-overlay hashes. It does not compare +rendered output, apply a final render release bump, or commit changes. +Render history is read from +`//` inside the +project repository, independently of `--output-dir`. The configured root defaults +to `specs/`. This path never generates synthetic history. Fetched upstream Git +metadata is retained through preparation; `--without-git` removes `.git` after +successful preparation. On failure, any remaining Git metadata is left for inspection. + +The source-preparation flags also apply to lockless components: + +| Flag | Behavior | +|------|----------| +| `--without-git` | Remove `.git` from prepared output after successful preparation. It remains available for upstream release/changelog preparation; no synthetic history is generated with or without the flag. | +| `--skip-overlays` | Skip configured and generated overlays, macro-file creation/loading, and `sources` metadata updates. Source acquisition still runs unless `--skip-sources` is also set. Upstream release/changelog preparation still runs. | +| `--skip-sources` | Skip upstream lookaside downloads, configured source downloads/generation, and archive overlays. Spec/sidecars are still fetched; other overlays and `sources` metadata updates still run unless `--skip-overlays` is set. Overlays requiring missing payloads can fail. | +| `--allow-no-hashes` | Compute missing hashes from acquired files when updating `sources`, defaulting to SHA-512 when `hash-type` is omitted. This does not edit TOML. With `--skip-sources`, missing hashes cannot be computed and cause an error; with `--skip-overlays`, metadata updates and hash computation are skipped. | + +Use `azldev component dist-git-repo -o ` to obtain a committed +working dist-git repository. That extraction command does not download archives. ## Configure the Project diff --git a/docs/user/reference/cli/azldev_component_prepare-sources.md b/docs/user/reference/cli/azldev_component_prepare-sources.md index 59912f1a7..d877c7fe8 100644 --- a/docs/user/reference/cli/azldev_component_prepare-sources.md +++ b/docs/user/reference/cli/azldev_component_prepare-sources.md @@ -14,8 +14,16 @@ for inspection or manual building. This is useful for verifying that overlays apply cleanly before running a full build. Only one component may be selected at a time. -Lockless components are unsupported; use 'azldev component dist-git-repo' instead -to extract their committed rendered files and history. +For lockless components, render-style preparation writes directly to '--output-dir' +and preserves fetched upstream Git metadata. Nonempty output requires '--force'. +It downloads/generates source payloads before macro-file setup and configured +overlays. Archive overlays run before spec and loose-file overlays. It does not +compare rendered output, apply a final render release bump, or commit changes. +Render history comes from 'project.rendered-specs-dir'//, +independently of '--output-dir'. No synthetic history is generated. +'--without-git' removes fetched Git metadata after successful preparation. +It honors '--skip-overlays', '--skip-sources', and '--allow-no-hashes'. +Use 'azldev component dist-git-repo' to extract committed rendered history. ``` azldev component prepare-sources [flags] @@ -46,7 +54,7 @@ azldev component prepare-sources [flags] --skip-overlays skip applying overlays to prepared sources --skip-sources skip downloading fetched sources when preparing the package (useful to extract dist-git metadata when source files are not needed) -s, --spec-path stringArray Spec path - --without-git Disable dist-git repository creation (enabled by default) + --without-git Exclude Git metadata from prepared output (locked components also skip synthetic dist-git creation) ``` ### Options inherited from parent commands diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 4d251762f..b9ffd5f9e 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -207,7 +207,10 @@ func TestSkillsDocumentLocklessBuildAndQuery(t *testing.T) { doc, err := catalog.SkillDocument("azldev", testParams()) require.NoError(t, err) assert.Contains(t, doc, "`comp query` extracts committed rendered history") - assert.Contains(t, doc, "`comp prepare-sources` rejects lockless components") + assert.Contains(t, doc, "Lockless `comp prepare-sources` writes directly to `--output-dir`") + assert.Contains(t, doc, "removes `.git` after successful preparation, preserving it on failure.") + assert.Contains(t, doc, "It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`.") + assert.Contains(t, doc, "from `--output-dir`") assert.Contains(t, doc, "locked queries still read the") doc, err = catalog.SkillDocument("azldev-build-component", testParams()) require.NoError(t, err) diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index 31305d40c..f51150e6b 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -142,8 +142,21 @@ Existing output requires `--force`, which deletes it after preflight checks. For lockless components, `comp query` extracts committed rendered history into temporary dist-git repositories and processes it with rpmautospec in Mock before querying. Uncommitted rendered edits are ignored; locked queries still read the -working tree. `comp prepare-sources` rejects lockless components; use -`comp dist-git-repo` instead. `comp history` continues to skip lockless components. +working tree. Lockless `comp prepare-sources` writes directly to `--output-dir` +using render-style preparation. Nonempty output requires `--force`; failures can +leave partial output. It downloads/generates sources before macro-file setup and +overlays, matching locked preparation. Archive overlays run before spec and +loose-file overlays. It does not compare +rendered output, apply a final render release bump, or commit. It reads render history from +`project.rendered-specs-dir` using `/`, never +from `--output-dir`. It never generates synthetic history. `--without-git` +removes `.git` after successful preparation, preserving it on failure. +It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. +Skipping overlays also skips macro-file setup and `sources` metadata updates; +skipping sources also skips archive overlays. Missing hashes cannot be computed +when source acquisition is skipped. +Use `comp dist-git-repo` for committed history extraction. +`comp history` continues to skip lockless components. The hidden `advanced` group (`adv`) contains specialist integrations such as MCP and mock helpers; it is intentionally omitted from normal help. diff --git a/internal/app/azldev/agentskill/content/build-component.md.tmpl b/internal/app/azldev/agentskill/content/build-component.md.tmpl index 0a29fb294..0e8142eb8 100644 --- a/internal/app/azldev/agentskill/content/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/build-component.md.tmpl @@ -28,7 +28,14 @@ with git-filter-repo; render and commit first. It downloads lookaside sources an builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` are rejected for lockless builds; `--no-check` is forwarded normally. -Lockless `comp prepare-sources` is unsupported; use `comp dist-git-repo` instead. +Lockless `comp prepare-sources` writes directly to `--output-dir` and downloads/generates +sources before macro-file setup, then applies archive overlays before spec and +loose-file overlays. History comes from `project.rendered-specs-dir`, not +`--output-dir`; no synthetic history is generated. `--without-git` removes `.git` +after successful preparation, preserving it on failure. +It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. +Nonempty output requires `--force`; failures can leave partial output. Use `comp dist-git-repo` +for committed history extraction. Locked builds retain their existing source preparation and flag behavior. investigate → modify → render → build → test → inspect diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 470d448da..cfb66916d 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -94,7 +94,14 @@ synthetic history still determine the number and order of bumps. Without the flag, no bumpspec runtime discovery occurs and the legacy static calculation is used for locked components. Lockless builds reject this flag and use Mock's rpmautospec plugin with committed dist-git history; lockless `comp prepare-sources` -is unsupported. The lockless upstream render workflow always runs +writes directly to `--output-dir` and downloads/generates sources before macro-file +setup, then applies archive overlays before spec/loose-file overlays. Nonempty output +requires `--force`; failures can leave partial output. It does not commit changes. +It reads history from `project.rendered-specs-dir`, not +`--output-dir`, without generating synthetic history. `--without-git` removes `.git` +after successful preparation, preserving it on failure. +It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. +The lockless upstream render workflow always runs host `rpmdev-bumpspec -c` only for non-autorelease specs and automatically commits the component output. Missing `--changelog-comment` opens `$EDITOR` or a local editor before output replacement, including for autorelease commits. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index a9f1a31ac..8963aab72 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -141,8 +141,21 @@ Existing output requires `--force`, which deletes it after preflight checks. For lockless components, `comp query` extracts committed rendered history into temporary dist-git repositories and processes it with rpmautospec in Mock before querying. Uncommitted rendered edits are ignored; locked queries still read the -working tree. `comp prepare-sources` rejects lockless components; use -`comp dist-git-repo` instead. `comp history` continues to skip lockless components. +working tree. Lockless `comp prepare-sources` writes directly to `--output-dir` +using render-style preparation. Nonempty output requires `--force`; failures can +leave partial output. It downloads/generates sources before macro-file setup and +overlays, matching locked preparation. Archive overlays run before spec and +loose-file overlays. It does not compare +rendered output, apply a final render release bump, or commit. It reads render history from +`project.rendered-specs-dir` using `/`, never +from `--output-dir`. It never generates synthetic history. `--without-git` +removes `.git` after successful preparation, preserving it on failure. +It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. +Skipping overlays also skips macro-file setup and `sources` metadata updates; +skipping sources also skips archive overlays. Missing hashes cannot be computed +when source acquisition is skipped. +Use `comp dist-git-repo` for committed history extraction. +`comp history` continues to skip lockless components. The hidden `advanced` group (`adv`) contains specialist integrations such as MCP and mock helpers; it is intentionally omitted from normal help. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl index 255ac6abd..dc4b18b8d 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl @@ -28,7 +28,14 @@ with git-filter-repo; render and commit first. It downloads lookaside sources an builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` are rejected for lockless builds; `--no-check` is forwarded normally. -Lockless `comp prepare-sources` is unsupported; use `comp dist-git-repo` instead. +Lockless `comp prepare-sources` writes directly to `--output-dir` and downloads/generates +sources before macro-file setup, then applies archive overlays before spec and +loose-file overlays. History comes from `project.rendered-specs-dir`, not +`--output-dir`; no synthetic history is generated. `--without-git` removes `.git` +after successful preparation, preserving it on failure. +It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. +Nonempty output requires `--force`; failures can leave partial output. Use `comp dist-git-repo` +for committed history extraction. Locked builds retain their existing source preparation and flag behavior. investigate → modify → render → build → test → inspect diff --git a/internal/app/azldev/cmds/component/build_lockless_internal_test.go b/internal/app/azldev/cmds/component/build_lockless_internal_test.go index 8f71d2d9b..254dde40e 100644 --- a/internal/app/azldev/cmds/component/build_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/build_lockless_internal_test.go @@ -11,7 +11,6 @@ import ( "strings" "testing" - "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/workdir" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" @@ -112,22 +111,3 @@ func TestLocklessBuildUsesDistGitAndCleansUp(t *testing.T) { }) } } - -func TestLocklessPrepareSourcesPreservesOutput(t *testing.T) { - for _, options := range []PrepareSourcesOptions{ - {}, {Force: true}, {SkipOverlays: true}, {WithoutGitRepo: true}, {RPMDevBumpspec: true}, - } { - env := testutils.NewTestEnv(t) - env.Config.LocklessComponents = map[string]bool{"curl": true} - env.Config.Components["curl"] = projectconfig.ComponentConfig{Name: "curl"} - options.ComponentFilter = components.ComponentFilter{ComponentNamePatterns: []string{"curl"}} - options.OutputDir = "/output" - - require.NoError(t, fileutils.WriteFile(env.FS(), "/output/keep", []byte("original"), fileperms.PublicFile)) - err := PrepareComponentSources(env.Env, &options) - require.ErrorContains(t, err, "use 'azldev component dist-git-repo' instead") - content, err := fileutils.ReadFile(env.FS(), "/output/keep") - require.NoError(t, err) - require.Equal(t, "original", string(content)) - } -} diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index d2ee899e9..59357223b 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -50,8 +50,16 @@ for inspection or manual building. This is useful for verifying that overlays apply cleanly before running a full build. Only one component may be selected at a time. -Lockless components are unsupported; use 'azldev component dist-git-repo' instead -to extract their committed rendered files and history.`, +For lockless components, render-style preparation writes directly to '--output-dir' +and preserves fetched upstream Git metadata. Nonempty output requires '--force'. +It downloads/generates source payloads before macro-file setup and configured +overlays. Archive overlays run before spec and loose-file overlays. It does not +compare rendered output, apply a final render release bump, or commit changes. +Render history comes from 'project.rendered-specs-dir'//, +independently of '--output-dir'. No synthetic history is generated. +'--without-git' removes fetched Git metadata after successful preparation. +It honors '--skip-overlays', '--skip-sources', and '--allow-no-hashes'. +Use 'azldev component dist-git-repo' to extract committed rendered history.`, Example: ` # Prepare sources for a component azldev component prep-sources -p curl -o ./build/work/scratch/curl --force @@ -77,7 +85,7 @@ to extract their committed rendered files and history.`, cmd.Flags().BoolVar(&options.SkipOverlays, "skip-overlays", false, "skip applying overlays to prepared sources") cmd.Flags().BoolVar(&options.WithoutGitRepo, "without-git", false, - "Disable dist-git repository creation (enabled by default)") + "Exclude Git metadata from prepared output (locked components also skip synthetic dist-git creation)") addRPMDevBumpspecFlag(cmd, &options.RPMDevBumpspec) cmd.Flags().BoolVar(&options.Force, "force", false, "delete and recreate the output directory if it already exists") @@ -91,10 +99,6 @@ to extract their committed rendered files and history.`, } func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) error { - if err := validatePrepareSourcesOptions(options); err != nil { - return err - } - var comps *components.ComponentSet resolver := components.NewResolver(env) @@ -116,9 +120,13 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er component := comps.Components()[0] - if env.ComponentWithoutLockfile(component.GetName()) { - return fmt.Errorf("component %#q uses the lockless workflow; use 'azldev component dist-git-repo' instead", - component.GetName()) + withoutLockfile := env.ComponentWithoutLockfile(component.GetName()) + if err := validatePrepareSourcesOptions(options, withoutLockfile); err != nil { + return err + } + + if withoutLockfile { + return prepareLocklessSources(env, component, options) } event := env.StartEvent("Preparing sources", "component", component.GetName(), "outputDir", options.OutputDir) @@ -198,12 +206,12 @@ func buildPreparerOptions( return opts } -func validatePrepareSourcesOptions(options *PrepareSourcesOptions) error { +func validatePrepareSourcesOptions(options *PrepareSourcesOptions, withoutLockfile bool) error { if !options.RPMDevBumpspec { return nil } - if options.WithoutGitRepo { + if options.WithoutGitRepo && !withoutLockfile { return errors.New("'--rpmdev-bumpspec' cannot be used with '--without-git'") } diff --git a/internal/app/azldev/cmds/component/preparesources_lockless.go b/internal/app/azldev/cmds/component/preparesources_lockless.go new file mode 100644 index 000000000..78131322b --- /dev/null +++ b/internal/app/azldev/cmds/component/preparesources_lockless.go @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +// prepareLocklessSources prepares render-style sources directly in the requested output directory. +func prepareLocklessSources( + env *azldev.Env, component components.Component, options *PrepareSourcesOptions, +) error { + if err := validateOutputDir(options.OutputDir); err != nil { + return err + } + + if err := fileutils.ValidateFilename(component.GetName()); err != nil { + return fmt.Errorf("invalid component name for source preparation:\n%w", err) + } + + event := env.StartEvent("Preparing lockless sources", "component", component.GetName(), "outputDir", options.OutputDir) + defer event.End() + + distro, err := sourceproviders.ResolveDistro(env, component) + if err != nil { + return fmt.Errorf("resolving distro for %#q:\n%w", component.GetName(), err) + } + + sourceManager, err := sourceproviders.NewSourceManager(env, distro) + if err != nil { + return fmt.Errorf("creating source manager for %#q:\n%w", component.GetName(), err) + } + + if err := CheckOutputDir(env, options); err != nil { + return err + } + + preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, + locklessSourcePreparerOptions(env, component, distro, options)...) + if err != nil { + return fmt.Errorf("creating source preparer for %#q:\n%w", component.GetName(), err) + } + + if err := preparer.PrepareSources(env, component, options.OutputDir, !options.SkipOverlays); err != nil { + return fmt.Errorf("preparing sources for %#q:\n%w", component.GetName(), err) + } + + if _, err := findSpecFile(env.FS(), options.OutputDir, component.GetName()); err != nil { + return fmt.Errorf("finding prepared spec:\n%w", err) + } + + if options.WithoutGitRepo { + if err := env.FS().RemoveAll(filepath.Join(options.OutputDir, ".git")); err != nil { + return fmt.Errorf("removing prepared Git metadata:\n%w", err) + } + } + + return nil +} + +func locklessSourcePreparerOptions( + env *azldev.Env, component components.Component, distro sourceproviders.ResolvedDistro, options *PrepareSourcesOptions, +) []sources.PreparerOption { + preparerOpts := append(gitRepoPreparerOptionsForMode(env, distro, true), + sources.WithSpecEditor(options.specEditorMode()), + sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), + sources.WithoutSyntheticHistory(), + sources.WithSkipLocalSpecConfig(), + sources.WithAutoreleaseResolver(hostRenderProcessor{env: env}), + sources.WithFetchedSources(func(_ context.Context, fetched components.Component, dir string) error { + //nolint:contextcheck // The callback context is the captured preparation env. + return prepareFetchedLocklessSources(env, fetched, dir, options.specEditorMode()) + }), + ) + if options.SkipSources { + preparerOpts = append(preparerOpts, sources.WithSkipLookaside()) + } + + if options.AllowNoHashes { + preparerOpts = append(preparerOpts, sources.WithAllowNoHashes()) + } + + if options.RPMDevBumpspec && component.GetConfig().Spec.SourceType != projectconfig.SpecSourceTypeLocal { + preparerOpts = append(preparerOpts, sources.WithRPMDevBumpspec(env, env.WorkDir(), "")) + } + + return preparerOpts +} + +func prepareFetchedLocklessSources( + env *azldev.Env, component components.Component, dir string, editor spec.EditorMode, +) error { + repo, root, err := openRenderRepo(env) + if err != nil { + return err + } + + path, err := distGitRepoSubdirectory(env, root, component.GetName()) + if err != nil { + return fmt.Errorf("locating committed render history:\n%w", err) + } + + if component.GetConfig().Spec.SourceType == projectconfig.SpecSourceTypeLocal { + return nil + } + + history, err := readLastRender(env, repo, root, path, component.GetName()) + if err != nil { + return fmt.Errorf("reading committed render state:\n%w", err) + } + + _, opened, err := openLocklessSpec(env, dir, component.GetName(), editor) + if err != nil { + return err + } + + version, err := opened.GetLastTag("", "Version") + if err != nil { + return fmt.Errorf("reading upstream Version:\n%w", err) + } + + update := &locklessRenderUpdate{repoRoot: root, path: path} + if err := collectRenderUpstreamChanges(env, dir, component.GetName(), version, history, update); err != nil { + return err + } + + if err := prepareLocklessSpec(env, hostRenderProcessor{env: env}, + dir, component.GetName(), editor, history); err != nil { + return fmt.Errorf("preparing changelog and release for %#q:\n%w", component.GetName(), err) + } + + return nil +} diff --git a/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go b/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go new file mode 100644 index 000000000..908792996 --- /dev/null +++ b/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go @@ -0,0 +1,554 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "context" + "errors" + "os/exec" + "path/filepath" + "strings" + "testing" + + gogit "github.com/go-git/go-git/v5" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestLocklessPrepareSourcesPreparesDirectlyInOutput(t *testing.T) { + const autorelease = "%autorelease" + + for _, testCase := range []struct { + name string + sourceType projectconfig.SpecSourceType + autorelease bool + failFetch bool + failPayload bool + failArchive bool + withoutGit bool + bumpspec bool + }{ + {name: "local", sourceType: projectconfig.SpecSourceTypeLocal}, + {name: "local without git", sourceType: projectconfig.SpecSourceTypeLocal, withoutGit: true}, + { + name: "without git and bumpspec", sourceType: projectconfig.SpecSourceTypeLocal, + withoutGit: true, bumpspec: true, + }, + {name: "upstream", sourceType: projectconfig.SpecSourceTypeUpstream}, + {name: "upstream autorelease", sourceType: projectconfig.SpecSourceTypeUpstream, autorelease: true}, + { + name: "upstream autorelease without git", sourceType: projectconfig.SpecSourceTypeUpstream, + autorelease: true, withoutGit: true, + }, + {name: "fetch failure", sourceType: projectconfig.SpecSourceTypeLocal, failFetch: true}, + {name: "payload failure", sourceType: projectconfig.SpecSourceTypeLocal, failPayload: true}, + { + name: "payload failure without git", sourceType: projectconfig.SpecSourceTypeLocal, + failPayload: true, withoutGit: true, + }, + {name: "archive overlay failure", sourceType: projectconfig.SpecSourceTypeLocal, failArchive: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + hash := strings.Repeat("a", 128) + + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", + Spec: projectconfig.SpecSource{SourceType: testCase.sourceType}, + Build: projectconfig.ComponentBuildConfig{ + Defines: map[string]string{"prepare_fixture": "ready"}, + }, + SourceFiles: []projectconfig.SourceFileReference{{ + Filename: "extra-source.tar.gz", Hash: hash, HashType: fileutils.HashTypeSHA512, + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://example.invalid/source"}, + }}, + Overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Prepared summary"}, + { + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, + Filename: "generated-input.txt", Regex: "downloaded", Replacement: "overlaid", + }, + }, + } + if testCase.failArchive { + config := env.Config.Components["curl"] + config.SourceFiles[0].Origin = projectconfig.Origin{Type: projectconfig.OriginTypeOverlay} + config.Overlays = append(config.Overlays, projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayRemoveFile, Archive: "extra-source.tar.gz", Filename: "remove-me.txt", + }) + env.Config.Components["curl"] = config + } + + repo, _ := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{"README": []byte("project")}}}) + originalRepo, originalPreparer := openRenderRepo, newSourcePreparer + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { + return repo, renderTestProjectRoot, nil + } + + t.Cleanup(func() { + openRenderRepo = originalRepo + newSourcePreparer = originalPreparer + }) + + ctrl := gomock.NewController(t) + manager := sourceproviders_test.NewMockSourceManager(ctrl) + + var preparedDir string + + fetchErr := errors.New("injected fetch failure") + payloadErr := errors.New("injected payload failure") + + release := "7%{?dist}" + if testCase.autorelease { + release = autorelease + } + + manager.EXPECT().FetchComponent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, comp components.Component, dir string, opts ...sourceproviders.FetchComponentOption) error { + preparedDir = dir + + require.Equal(t, "curl", comp.GetName()) + stageLocklessFetchedSources(t, env.FS(), dir, release, opts) + + if testCase.failFetch { + return fetchErr + } + + return nil + }) + + if !testCase.failFetch { + manager.EXPECT().FetchFiles(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string) error { + require.Equal(t, preparedDir, dir) + stageLocklessPreparePayloads(t, env.FS(), dir) + + if testCase.failPayload { + return payloadErr + } + + return nil + }) + } + + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + env.CmdFactory.RegisterCommandInSearchPath("rpmautospec") + + var autospecCalls []string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + require.Equal(t, "rpmautospec", cmd.Args[0]) + require.True(t, testCase.autorelease) + require.Equal(t, preparedDir, cmd.Dir) + + autospecCalls = append(autospecCalls, cmd.Args[1]) + if cmd.Args[1] == "generate-changelog" { + return "* Upstream changelog\n", nil + } + + require.Equal(t, "calculate-release", cmd.Args[1]) + + return "4", nil + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("unexpected finalization command: %v", cmd.Args) + + return nil + } + + const output = "/external/output" + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(output, "keep"), + []byte("original output"), fileperms.PublicFile)) + err := PrepareComponentSources(env.Env, &PrepareSourcesOptions{ + ComponentFilter: components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}, + OutputDir: output, Force: true, + WithoutGitRepo: testCase.withoutGit, RPMDevBumpspec: testCase.bumpspec, + }) + + require.Equal(t, output, preparedDir) + + switch { + case testCase.failFetch: + require.ErrorIs(t, err, fetchErr) + case testCase.failPayload || testCase.failArchive: + if testCase.failPayload { + require.ErrorIs(t, err, payloadErr) + } else { + require.ErrorContains(t, err, "extra-source.tar.gz") + } + + specContent, err := fileutils.ReadFile(env.FS(), filepath.Join(preparedDir, "curl.spec")) + require.NoError(t, err) + assert.NotContains(t, string(specContent), "Summary: Prepared summary") + + content, err := fileutils.ReadFile(env.FS(), filepath.Join(preparedDir, "generated-input.txt")) + require.NoError(t, err) + assert.Equal(t, "downloaded", string(content)) + default: + require.NoError(t, err) + + wantRelease := release + if testCase.autorelease { + wantRelease = "%autorelease -b 4" + + assert.Equal(t, []string{"generate-changelog", "calculate-release"}, autospecCalls) + } + + assertLocklessPreparedSourceTree(t, env.FS(), preparedDir, wantRelease, hash) + } + + wantGit := !testCase.withoutGit || err != nil + for _, name := range []string{".git/HEAD", "unreferenced"} { + exists, err := fileutils.Exists(env.FS(), filepath.Join(preparedDir, name)) + require.NoError(t, err) + assert.Equal(t, name != ".git/HEAD" || wantGit, exists, "unexpected prepared file: %s", name) + } + + exists, err := fileutils.Exists(env.FS(), filepath.Join(output, "keep")) + require.NoError(t, err) + assert.False(t, exists, "--force must remove existing output") + + exists, err = fileutils.Exists(env.FS(), filepath.Join(preparedDir, "extra-source.tar.gz")) + require.NoError(t, err) + assert.Equal(t, !testCase.failFetch, exists) + exists, err = fileutils.Exists(env.FS(), filepath.Join(output, "c/curl")) + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + +func TestLocklessPrepareSourcesRejectsNonemptyOutputWithoutForce(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + } + + const output = "/external/output" + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(output, "keep"), + []byte("original output"), fileperms.PublicFile)) + + originalPreparer := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, _ opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + t.Fatal("nonempty output must be rejected before source preparation") + + return nil, errors.New("unexpected source preparer construction") + } + + t.Cleanup(func() { newSourcePreparer = originalPreparer }) + + err := PrepareComponentSources(env.Env, &PrepareSourcesOptions{ + ComponentFilter: components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}, + OutputDir: output, + }) + require.ErrorContains(t, err, "--force") + content, err := fileutils.ReadFile(env.FS(), filepath.Join(output, "keep")) + require.NoError(t, err) + assert.Equal(t, "original output", string(content)) +} + +func TestLocklessPrepareSourcesFlags(t *testing.T) { + const helloSHA256 = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + + for _, testCase := range []struct { + name string + options PrepareSourcesOptions + missingHash bool + missingType bool + archive bool + wantError string + }{ + {name: "skip overlays", options: PrepareSourcesOptions{SkipOverlays: true}}, + {name: "skip sources", options: PrepareSourcesOptions{SkipSources: true}, archive: true}, + {name: "skip both", options: PrepareSourcesOptions{SkipOverlays: true, SkipSources: true}}, + {name: "allow missing hash", options: PrepareSourcesOptions{AllowNoHashes: true}, missingHash: true}, + { + name: "default hash algorithm", options: PrepareSourcesOptions{AllowNoHashes: true}, + missingHash: true, missingType: true, + }, + {name: "missing hash rejected", missingHash: true, wantError: "missing required 'hash'"}, + { + name: "missing hash needs payload", options: PrepareSourcesOptions{AllowNoHashes: true, SkipSources: true}, + missingHash: true, wantError: "downloads were skipped", + }, + { + name: "skip overlays leaves missing hash unresolved", + options: PrepareSourcesOptions{AllowNoHashes: true, SkipOverlays: true}, missingHash: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + ref := projectconfig.SourceFileReference{ + Filename: "extra-source.tar.gz", Hash: helloSHA256, HashType: fileutils.HashTypeSHA256, + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://example.invalid/source"}, + } + if testCase.missingHash { + ref.Hash = "" + } + + if testCase.missingType { + ref.HashType = "" + } + + env, dir, err := runLocklessPrepareFlagFixture(t, testCase.options, ref, testCase.archive) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + + return + } + + require.NoError(t, err) + exists, err := fileutils.Exists(env.FS(), filepath.Join(dir, ref.Filename)) + require.NoError(t, err) + assert.Equal(t, !testCase.options.SkipSources, exists) + exists, err = fileutils.Exists(env.FS(), filepath.Join(dir, "curl.azl.macros")) + require.NoError(t, err) + assert.Equal(t, !testCase.options.SkipOverlays, exists) + + content, err := fileutils.ReadFile(env.FS(), filepath.Join(dir, "sources")) + require.NoError(t, err) + specContent, err := fileutils.ReadFile(env.FS(), filepath.Join(dir, "curl.spec")) + require.NoError(t, err) + + if testCase.options.SkipOverlays { + assert.Equal(t, "# upstream metadata\n", string(content)) + assert.Equal(t, renderTestSpec("1", "7", ""), string(specContent)) + } else { + assert.Contains(t, string(specContent), "Summary: Prepared summary\n") + + hash, hashType := helloSHA256, fileutils.HashTypeSHA256 + if testCase.missingType { + hashType = fileutils.HashTypeSHA512 + hash = "309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f" + + "989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f" + } + + assert.Contains(t, string(content), string(hashType)+" ("+ref.Filename+") = "+hash) + } + }) + } +} + +func runLocklessPrepareFlagFixture( + t *testing.T, options PrepareSourcesOptions, ref projectconfig.SourceFileReference, archiveOverlay bool, +) (env *testutils.TestEnv, dir string, err error) { + t.Helper() + + env = testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + + config := projectconfig.ComponentConfig{ + Name: "curl", Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + Build: projectconfig.ComponentBuildConfig{Defines: map[string]string{"prepare_fixture": "ready"}}, + SourceFiles: []projectconfig.SourceFileReference{ref}, + Overlays: []projectconfig.ComponentOverlay{{ + Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Prepared summary", + }}, + } + if archiveOverlay { + config.Overlays = append(config.Overlays, projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayRemoveFile, Archive: "upstream.tar.gz", Filename: "vendor/**", + }) + config.SourceFiles = append(config.SourceFiles, projectconfig.SourceFileReference{ + Filename: "upstream.tar.gz", Hash: strings.Repeat("a", 64), HashType: fileutils.HashTypeSHA256, + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeOverlay}, + }) + } + + env.Config.Components["curl"] = config + repo, _ := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{"README": []byte("project")}}}) + originalRepo, originalPreparer := openRenderRepo, newSourcePreparer + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { + return repo, renderTestProjectRoot, nil + } + + t.Cleanup(func() { + openRenderRepo = originalRepo + newSourcePreparer = originalPreparer + }) + manager := sourceproviders_test.NewMockSourceManager(gomock.NewController(t)) + manager.EXPECT().FetchComponent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func( + _ context.Context, _ components.Component, fetchedDir string, opts ...sourceproviders.FetchComponentOption, + ) error { + dir = fetchedDir + + var resolved sourceproviders.FetchComponentOptions + for _, opt := range opts { + opt(&resolved) + } + + assert.Equal(t, options.SkipSources, resolved.SkipLookaside) + assert.True(t, resolved.PreserveGitDir) + assert.True(t, resolved.SkipLocalSpecConfig) + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("1", "7", "")), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(dir, ".git/HEAD"), + []byte("upstream history"), fileperms.PublicFile)) + + return fileutils.WriteFile(env.FS(), filepath.Join(dir, "sources"), + []byte("# upstream metadata\n"), fileperms.PublicFile) + }) + + if !options.SkipSources { + acquire := func(_ context.Context, _ components.Component, fetchedDir string) error { + exists, err := fileutils.Exists(env.FS(), filepath.Join(fetchedDir, "curl.azl.macros")) + require.NoError(t, err) + assert.False(t, exists, "payload acquisition must precede macro-file setup") + + return fileutils.WriteFile(env.FS(), filepath.Join(fetchedDir, ref.Filename), + []byte("hello world"), fileperms.PublicFile) + } + manager.EXPECT().FetchFiles(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(acquire) + } + + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + opts ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, opts...) + } + options.OutputDir = "/external/output" + options.ComponentFilter = components.ComponentFilter{ComponentNamePatterns: []string{"curl"}} + err = PrepareComponentSources(env.Env, &options) + + return env, dir, err +} + +func TestLocklessPrepareSourcesUsesConfiguredRenderHistory(t *testing.T) { + for _, testCase := range []struct { + name, configuredDir, historyPath, wantError string + }{ + {name: "default", historyPath: "specs/c/Curl"}, + {name: "relative", configuredDir: "custom-specs", historyPath: "custom-specs/c/Curl"}, + {name: "absolute", configuredDir: "/project/SPECS", historyPath: "SPECS/c/Curl"}, + {name: "outside repository", configuredDir: "/external/specs", wantError: "escapes repository root"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.Project.RenderedSpecsDir = testCase.configuredDir + + files := map[string][]byte{"README": []byte("project")} + if testCase.historyPath != "" { + files[testCase.historyPath+"/Curl.spec"] = []byte(renderTestSpec("1", "9", "")) + } + + repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: files}}) + originalRepo := openRenderRepo + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { + return repo, renderTestProjectRoot, nil + } + + t.Cleanup(func() { openRenderRepo = originalRepo }) + + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + component.EXPECT().GetName().AnyTimes().Return("Curl") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + }) + + historyErr := errors.New("injected history lookup failure") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{ + "git", "-C", renderTestProjectRoot, "--literal-pathspecs", "log", "-1", "--format=%H", + "--end-of-options", hashes[0], "--", testCase.historyPath + "/", + }, cmd.Args) + + return "", historyErr + } + + err := prepareFetchedLocklessSources(env.Env, component, "/work/staging", spec.EditorLegacy) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + } else { + require.ErrorIs(t, err, historyErr) + } + }) + } +} + +func assertLocklessPreparedSourceTree(t *testing.T, fs opctx.FS, dir, release, hash string) { + t.Helper() + + specContent, err := fileutils.ReadFile(fs, filepath.Join(dir, "curl.spec")) + require.NoError(t, err) + assert.Contains(t, string(specContent), "Summary: Prepared summary\n") + assert.Equal(t, 1, strings.Count(string(specContent), "%{load:")) + assert.Contains(t, string(specContent), "Release: "+release+"\n") + + content, err := fileutils.ReadFile(fs, filepath.Join(dir, "generated-input.txt")) + require.NoError(t, err) + assert.Equal(t, "overlaid", string(content)) + + manifest, err := fileutils.ReadFile(fs, filepath.Join(dir, "sources")) + require.NoError(t, err) + assert.Contains(t, string(manifest), "SHA512 (extra-source.tar.gz) = "+hash) +} + +func stageLocklessFetchedSources( + t *testing.T, fs opctx.FS, dir, release string, opts []sourceproviders.FetchComponentOption, +) { + t.Helper() + + var resolved sourceproviders.FetchComponentOptions + for _, opt := range opts { + opt(&resolved) + } + + assert.True(t, resolved.PreserveGitDir) + assert.False(t, resolved.SkipLookaside) + assert.True(t, resolved.SkipLocalSpecConfig) + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("1", release, "%autochangelog\n")), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, ".git/HEAD"), + []byte("upstream history"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, "unreferenced"), + []byte("keep sidecar"), fileperms.PublicFile)) +} + +func stageLocklessPreparePayloads(t *testing.T, fs opctx.FS, dir string) { + t.Helper() + + head, err := fileutils.ReadFile(fs, filepath.Join(dir, ".git/HEAD")) + require.NoError(t, err) + assert.Equal(t, "upstream history", string(head), "Git metadata must remain during preparation") + + specContent, err := fileutils.ReadFile(fs, filepath.Join(dir, "curl.spec")) + require.NoError(t, err) + assert.NotContains(t, string(specContent), "%{load:") + assert.NotContains(t, string(specContent), "Source9999:") + assert.NotContains(t, string(specContent), "Summary: Prepared summary") + + exists, err := fileutils.Exists(fs, filepath.Join(dir, "curl.azl.macros")) + require.NoError(t, err) + assert.False(t, exists, "payload acquisition must precede macro-file setup") + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, "extra-source.tar.gz"), + []byte("source payload"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, "generated-input.txt"), + []byte("downloaded"), fileperms.PublicFile)) +} diff --git a/internal/app/azldev/cmds/component/preparesources_test.go b/internal/app/azldev/cmds/component/preparesources_test.go index b7d8bc46d..b47f57878 100644 --- a/internal/app/azldev/cmds/component/preparesources_test.go +++ b/internal/app/azldev/cmds/component/preparesources_test.go @@ -7,7 +7,9 @@ import ( "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/stretchr/testify/assert" @@ -63,8 +65,10 @@ func TestPrepareSourcesCmd_NoMatch(t *testing.T) { require.Error(t, err) } -func TestPrepareSourcesOptions_RejectRPMDevBumpspecInvalidCombinationsBeforeDiscovery(t *testing.T) { +func TestPrepareSourcesOptions_RejectRPMDevBumpspecInvalidCombinations(t *testing.T) { testEnv := testutils.NewTestEnv(t) + testEnv.Config.Components["curl"] = projectconfig.ComponentConfig{Name: "curl"} + for _, testCase := range []struct { name string options component.PrepareSourcesOptions @@ -82,6 +86,9 @@ func TestPrepareSourcesOptions_RejectRPMDevBumpspecInvalidCombinationsBeforeDisc }, } { t.Run(testCase.name, func(t *testing.T) { + testCase.options.ComponentFilter = components.ComponentFilter{ + ComponentNamePatterns: []string{"curl"}, SkipLockValidation: true, + } err := component.PrepareComponentSources(testEnv.Env, &testCase.options) require.EqualError(t, err, testCase.message) }) diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 8bc000777..c0af3dc47 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -389,7 +389,7 @@ func (p *sourcePreparerImpl) PrepareSources( func (p *sourcePreparerImpl) componentFetchOptions(applyOverlays bool) []sourceproviders.FetchComponentOption { var options []sourceproviders.FetchComponentOption - if applyOverlays && p.withGitRepo { + if p.withGitRepo && (applyOverlays || p.skipSyntheticHistory) { options = append(options, sourceproviders.WithPreserveGitDir()) } diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index 97bc67129..7889af784 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -190,7 +190,9 @@ func TestPrepareSources_ArchiveOverlayRehashesSourcesEntry(t *testing.T) { component.EXPECT().GetName().AnyTimes().Return(componentName) component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Build: projectconfig.ComponentBuildConfig{Defines: map[string]string{"prepare_fixture": "ready"}}, Overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Overlaid"}, { Type: projectconfig.ComponentOverlayRemoveFile, Archive: archiveName, @@ -208,14 +210,26 @@ func TestPrepareSources_ArchiveOverlayRehashesSourcesEntry(t *testing.T) { }, }) - // The archive and 'sources' file already exist on disk; the source manager - // only needs to provide the spec file (FetchFiles is a no-op download). - sourceManager.EXPECT().FetchFiles(gomock.Any(), component, outputDir).Return(nil) + // The archive and 'sources' file stand in for acquired upstream payloads. + sourceManager.EXPECT().FetchFiles(gomock.Any(), component, outputDir).DoAndReturn( + func(_ context.Context, _ components.Component, _ string) error { + content, err := fileutils.ReadFile(ctx.FS(), specPath) + require.NoError(t, err) + assert.NotContains(t, string(content), "%{load:") + assert.NotContains(t, string(content), "Summary: Overlaid") + + hash, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeMD5, archivePath) + require.NoError(t, err) + assert.Equal(t, originalHash, hash, "archive overlays must follow acquisition") + + return nil + }) + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, outputDir, gomock.Any()).DoAndReturn( func(_ interface{}, _ interface{}, dir string, _ ...sourceproviders.FetchComponentOption) error { return fileutils.WriteFile( ctx.FS(), filepath.Join(dir, componentName+".spec"), - []byte("# test spec"), fileperms.PublicFile) + []byte("Name: test-component\nVersion: 1.0\nRelease: 1\nSummary: Original\n"), fileperms.PublicFile) }, ) @@ -245,6 +259,15 @@ func TestPrepareSources_ArchiveOverlayRehashesSourcesEntry(t *testing.T) { assert.Equal(t, fileutils.HashTypeSHA512, entry.HashType, "configured post-overlay hash type must be used") assert.Equal(t, repackedHash, entry.Hash, "'sources' entry must record the repacked archive's hash") assert.NotEqual(t, originalHash, entry.Hash, "'sources' entry hash must have been updated") + + extractedDir := t.TempDir() + require.NoError(t, archive.ExtractAuto(archivePath, extractedDir)) + assert.FileExists(t, filepath.Join(extractedDir, "pkg-1.0/keep.txt")) + assert.NoFileExists(t, filepath.Join(extractedDir, "pkg-1.0/remove-me.txt")) + + specContent, err := fileutils.ReadFile(ctx.FS(), specPath) + require.NoError(t, err) + assert.Contains(t, string(specContent), "Summary: Overlaid") } func TestPrepareSources_ArchiveOverlayAssociationValidation(t *testing.T) { From 3d0630567db1863b2e51fa62817565e413c56d5d Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sat, 7 Nov 2026 06:51:45 +0000 Subject: [PATCH 47/70] fix(component): use working inputs when preparing lockless sources Resolve the current upstream commit and distro from working-tree TOML during lockless source preparation. Continue reading the previous render's configuration and component dist-git snapshot from committed history. Keep render's committed-input behavior unchanged. Cover modified and untracked TOML and overlay inputs, and verify that dirty dist-git specs and sidecars do not affect release/changelog baselines. Update user documentation, generated help and emitted agent skills. Validation: mage fix all, mage unit and mage docs passed. Quality checks in mage all passed; scenario tests and mage scenarioUpdate remain blocked by the existing Docker-provider/$XDG_RUNTIME_DIR initialization failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 5 + .../cli/azldev_component_prepare-sources.md | 3 + .../app/azldev/agentskill/agentskill_test.go | 2 + .../azldev/agentskill/content/azldev.md.tmpl | 5 +- .../content/withoutlockfile/azldev.md.tmpl | 5 +- .../azldev/cmds/component/preparesources.go | 3 + .../cmds/component/preparesources_lockless.go | 2 +- .../preparesources_lockless_internal_test.go | 193 ++++++++++++++++++ .../azldev/cmds/component/render_lockless.go | 30 ++- .../render_lockless_internal_test.go | 8 +- 10 files changed, 240 insertions(+), 16 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 04b66c060..95e9380cd 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -107,6 +107,11 @@ Mock's `--nocheck`; `--srpm-only` skips binary builds. Lockless builds reject `component prepare-sources` uses lockless render preparation while writing directly into `--output-dir`, allowing uncommitted and untracked project files. +Current TOML configuration (including upstream pins and distro settings), overlay +inputs, scripts, and local sources are read from the working tree. Modified and +newly added files are used without committing them first. Only the existing +component dist-git state used as the release/changelog baseline is read from +committed files; modified and untracked files in that directory are ignored. Nonempty output requires `--force`, which removes existing contents before preparation. Failures can leave partially prepared output. It fetches upstream lookaside payloads with the spec/sidecars, prepares the upstream release/changelog, diff --git a/docs/user/reference/cli/azldev_component_prepare-sources.md b/docs/user/reference/cli/azldev_component_prepare-sources.md index d877c7fe8..816398e8e 100644 --- a/docs/user/reference/cli/azldev_component_prepare-sources.md +++ b/docs/user/reference/cli/azldev_component_prepare-sources.md @@ -16,6 +16,9 @@ overlays apply cleanly before running a full build. Only one component may be selected at a time. For lockless components, render-style preparation writes directly to '--output-dir' and preserves fetched upstream Git metadata. Nonempty output requires '--force'. +Current TOML and file inputs use the working tree, including modified and untracked +files. Release/changelog baselines use only committed component dist-git files; +uncommitted edits in that existing dist-git directory are ignored. It downloads/generates source payloads before macro-file setup and configured overlays. Archive overlays run before spec and loose-file overlays. It does not compare rendered output, apply a final render release bump, or commit changes. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index b9ffd5f9e..45a173405 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -208,6 +208,8 @@ func TestSkillsDocumentLocklessBuildAndQuery(t *testing.T) { require.NoError(t, err) assert.Contains(t, doc, "`comp query` extracts committed rendered history") assert.Contains(t, doc, "Lockless `comp prepare-sources` writes directly to `--output-dir`") + assert.Contains(t, doc, "including modified and untracked files.") + assert.Contains(t, doc, "committed component dist-git files; edits in the existing dist-git directory are") assert.Contains(t, doc, "removes `.git` after successful preparation, preserving it on failure.") assert.Contains(t, doc, "It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`.") assert.Contains(t, doc, "from `--output-dir`") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index f51150e6b..c296616c7 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -143,7 +143,10 @@ For lockless components, `comp query` extracts committed rendered history into temporary dist-git repositories and processes it with rpmautospec in Mock before querying. Uncommitted rendered edits are ignored; locked queries still read the working tree. Lockless `comp prepare-sources` writes directly to `--output-dir` -using render-style preparation. Nonempty output requires `--force`; failures can +using render-style preparation. Current TOML and file inputs use the working tree, +including modified and untracked files. Release/changelog baselines use only +committed component dist-git files; edits in the existing dist-git directory are +ignored. Nonempty output requires `--force`; failures can leave partial output. It downloads/generates sources before macro-file setup and overlays, matching locked preparation. Archive overlays run before spec and loose-file overlays. It does not compare diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 8963aab72..033fd3b04 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -142,7 +142,10 @@ For lockless components, `comp query` extracts committed rendered history into temporary dist-git repositories and processes it with rpmautospec in Mock before querying. Uncommitted rendered edits are ignored; locked queries still read the working tree. Lockless `comp prepare-sources` writes directly to `--output-dir` -using render-style preparation. Nonempty output requires `--force`; failures can +using render-style preparation. Current TOML and file inputs use the working tree, +including modified and untracked files. Release/changelog baselines use only +committed component dist-git files; edits in the existing dist-git directory are +ignored. Nonempty output requires `--force`; failures can leave partial output. It downloads/generates sources before macro-file setup and overlays, matching locked preparation. Archive overlays run before spec and loose-file overlays. It does not compare diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index 59357223b..61c2e129b 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -52,6 +52,9 @@ overlays apply cleanly before running a full build. Only one component may be selected at a time. For lockless components, render-style preparation writes directly to '--output-dir' and preserves fetched upstream Git metadata. Nonempty output requires '--force'. +Current TOML and file inputs use the working tree, including modified and untracked +files. Release/changelog baselines use only committed component dist-git files; +uncommitted edits in that existing dist-git directory are ignored. It downloads/generates source payloads before macro-file setup and configured overlays. Archive overlays run before spec and loose-file overlays. It does not compare rendered output, apply a final render release bump, or commit changes. diff --git a/internal/app/azldev/cmds/component/preparesources_lockless.go b/internal/app/azldev/cmds/component/preparesources_lockless.go index 78131322b..08b14beef 100644 --- a/internal/app/azldev/cmds/component/preparesources_lockless.go +++ b/internal/app/azldev/cmds/component/preparesources_lockless.go @@ -115,7 +115,7 @@ func prepareFetchedLocklessSources( return nil } - history, err := readLastRender(env, repo, root, path, component.GetName()) + history, err := readLastRender(env, repo, root, path, component.GetName(), true) if err != nil { return fmt.Errorf("reading committed render state:\n%w", err) } diff --git a/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go b/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go index 908792996..e8e0004ef 100644 --- a/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go @@ -6,6 +6,7 @@ package component import ( "context" "errors" + "fmt" "os/exec" "path/filepath" "strings" @@ -272,6 +273,198 @@ func TestLocklessPrepareSourcesRejectsNonemptyOutputWithoutForce(t *testing.T) { assert.Equal(t, "original output", string(content)) } +func TestLocklessPrepareSourcesUsesWorkingInputs(t *testing.T) { + const fetchCommand = "fetch" + + for _, inputDir := range []string{"", "draft"} { + t.Run("inputs/"+inputDir, func(t *testing.T) { + env, repo, hashes := locklessWorkingInputsFixture(t, inputDir) + originalRepo, originalPreparer := openRenderRepo, newSourcePreparer + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { + return repo, renderTestProjectRoot, nil + } + + t.Cleanup(func() { + openRenderRepo = originalRepo + newSourcePreparer = originalPreparer + }) + + var upstreamRanges []string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + require.Equal(t, "git", cmd.Args[0]) + + switch cmd.Args[3] { + case "--literal-pathspecs": + assert.Equal(t, []string{ + "git", "-C", "/project", "--literal-pathspecs", "log", "-1", "--format=%H", + "--end-of-options", hashes[1], "--", "specs/c/curl/", + }, cmd.Args) + + return hashes[0], nil + case fetchCommand: + assert.Equal(t, "+refs/heads/f40:refs/remotes/origin/f40", cmd.Args[len(cmd.Args)-1]) + + return "", nil + case "rev-parse": + assert.Equal(t, "aaaaaaa^{commit}", cmd.Args[len(cmd.Args)-1]) + + return "aaaaaaa", nil + case "log": + upstreamRanges = append(upstreamRanges, cmd.Args[len(cmd.Args)-2]) + + return "ccccccc Working upstream change\n", nil + default: + t.Fatalf("unexpected preparation command: %v", cmd.Args) + + return "", errors.New("unexpected preparation command") + } + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("unexpected finalization command: %v", cmd.Args) + + return nil + } + + history, err := readLastRender(env.Env, repo, "/project", "specs/c/curl", "curl", true) + require.NoError(t, err) + assert.Equal(t, "aaaaaaa", history.previous) + assert.Equal(t, "ccccccc", history.current) + assert.Equal(t, "f40", history.previousBranch) + assert.Equal(t, "f42", history.currentBranch) + helper, err := history.tree.File("macros.inc") + require.NoError(t, err) + content, err := helper.Contents() + require.NoError(t, err) + assert.Equal(t, "committed helper", content) + + _, err = history.tree.File("untracked.txt") + require.Error(t, err) + + manager := sourceproviders_test.NewMockSourceManager(gomock.NewController(t)) + manager.EXPECT().FetchComponent(gomock.Any(), gomock.Any(), "/external/output", gomock.Any()).DoAndReturn( + func(_ context.Context, comp components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + assert.Equal(t, "ccccccc", comp.GetConfig().Spec.UpstreamCommit) + + return fileutils.WriteFile(env.FS(), filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("1", "1", "* Upstream changelog\n")), fileperms.PublicFile) + }) + manager.EXPECT().FetchFiles(gomock.Any(), gomock.Any(), "/external/output").Return(nil) + + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + require.NoError(t, PrepareComponentSources(env.Env, &PrepareSourcesOptions{ + ComponentFilter: components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}, + OutputDir: "/external/output", + })) + assert.Equal(t, []string{"aaaaaaa..ccccccc"}, upstreamRanges) + + prepared, err := fileutils.ReadFile(env.FS(), "/external/output/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(prepared), "Release: 9\n") + assert.Contains(t, string(prepared), "* Committed changelog\n") + assert.NotContains(t, string(prepared), "* Upstream changelog\n") + + copied, err := fileutils.ReadFile(env.FS(), "/external/output/overlay-input.txt") + require.NoError(t, err) + assert.Equal(t, "working overlay input", string(copied)) + + macros, err := fileutils.ReadFile(env.FS(), "/external/output/curl.azl.macros") + require.NoError(t, err) + assert.Contains(t, string(macros), "%working_input working\n") + }) + } +} + +func locklessWorkingInputsFixture( + t *testing.T, inputDir string, +) (env *testutils.TestEnv, repo *gogit.Repository, hashes []string) { + t.Helper() + + const ( + project = `[project] +default-distro = { name = "test", version = "3" } +[distros.test.versions."1"] +release-ver = "1" +dist-git-branch = "f40" +[distros.test.versions."2"] +release-ver = "2" +dist-git-branch = "f41" +[distros.test.versions."3"] +release-ver = "3" +dist-git-branch = "f42" +` + component = `[components.curl.spec] +type = 'upstream' +upstream-distro = { name = 'test', version = '1' } +[components.curl.build.defines] +working_input = 'committed' +[[components.curl.overlays]] +type = 'file-add' +file = 'overlay-input.txt' +source = 'input.txt' +` + ) + + root := "includes = [\"component.toml\", \"pins/*.toml\"]\n" + project + repo, hashes = testRepoWithCommits(t, []testRepoCommit{ + {files: map[string][]byte{ + "azldev.toml": []byte(root), + "lockless-components.txt": []byte("curl\n"), + "component.toml": []byte(component), + "pins/curl.toml": []byte("[components.curl.spec]\nupstream-commit = 'aaaaaaa'\n"), + "input.txt": []byte("committed overlay input"), + "specs/c/curl/curl.spec": []byte(renderTestSpec("1", "9", "* Committed changelog\n")), + "specs/c/curl/macros.inc": []byte("committed helper"), + }}, + {files: map[string][]byte{ + "component.toml": []byte(strings.Replace(component, "version = '1'", "version = '2'", 1)), + "pins/curl.toml": []byte("[components.curl.spec]\nupstream-commit = 'bbbbbbb'\n"), + }}, + }) + env = testutils.NewTestEnv(t) + tree, err := resolveTree(repo, hashes[1]) + require.NoError(t, err) + require.NoError(t, copyTreeToFS(tree, env.FS(), "/project")) + + workingComponent := strings.Replace(component, "version = '1'", "version = '3'", 1) + workingComponent = strings.Replace(workingComponent, "'committed'", "'working'", 1) + working := map[string][]byte{ + "azldev.toml": []byte(fmt.Sprintf("includes = [%q, %q]\n%s", + filepath.Join(inputDir, "component.toml"), filepath.Join(inputDir, "pins/*.toml"), project)), + filepath.Join(inputDir, "component.toml"): []byte(workingComponent), + filepath.Join(inputDir, "pins/curl.toml"): []byte("[components.curl.spec]\nupstream-commit = 'ccccccc'\n"), + filepath.Join(inputDir, "input.txt"): []byte("working overlay input"), + "specs/c/curl/curl.spec": []byte("invalid uncommitted dist-git spec"), + "specs/c/curl/macros.inc": []byte("uncommitted helper"), + "specs/c/curl/untracked.txt": []byte("untracked dist-git sidecar"), + } + worktree, err := repo.Worktree() + require.NoError(t, err) + + for path, content := range working { + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join("/project", path), content, fileperms.PublicFile)) + file, err := worktree.Filesystem.Create(path) + require.NoError(t, err) + _, err = file.Write(content) + require.NoError(t, err) + require.NoError(t, file.Close()) + } + + _, config, err := projectconfig.LoadProjectConfig(env.FS(), env.Env.OSEnv(), + "/project", true, "/work", nil, false, false) + require.NoError(t, err) + + *env.Config = *config + + return env, repo, hashes +} + func TestLocklessPrepareSourcesFlags(t *testing.T) { const helloSHA256 = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 257a66011..e2455ec8c 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -114,7 +114,7 @@ func prepareLocklessRender( return nil, err } - history, err := readLastRender(env, repo, update.repoRoot, update.path, comp.GetName()) + history, err := readLastRender(env, repo, update.repoRoot, update.path, comp.GetName(), false) if err != nil { return nil, fmt.Errorf("reading committed render state:\n%w", err) } @@ -172,7 +172,7 @@ func prepareLocklessRenderOutput( } func readLastRender( - env *azldev.Env, repo *gogit.Repository, root, path, name string, + env *azldev.Env, repo *gogit.Repository, root, path, name string, useWorkingInputs bool, ) (lastRender, error) { var history lastRender @@ -224,20 +224,26 @@ func readLastRender( return history, fmt.Errorf("reading last render commit:\n%w", err) } - err = readRenderedUpstreamCommits(env, root, name, tree, last, &history) + err = readRenderedUpstreamCommits(env, root, name, tree, last, &history, useWorkingInputs) return history, err } func readRenderedUpstreamCommits( env *azldev.Env, root, name string, tree *object.Tree, last *object.Commit, history *lastRender, + useWorkingInputs bool, ) error { projectPath, err := repoRelPath(root, env.ProjectDir()) if err != nil { return err } - history.current, history.currentBranch, err = renderUpstreamCommit(env, tree, projectPath, name, false) + if useWorkingInputs { + history.current, history.currentBranch, err = configuredRenderUpstreamCommit(env, name, false) + } else { + history.current, history.currentBranch, err = renderUpstreamCommit(env, tree, projectPath, name, false) + } + if err != nil { return fmt.Errorf("reading current render upstream commit:\n%w", err) } @@ -266,14 +272,20 @@ func renderUpstreamCommit( return "", "", err } - comp, err := components.NewResolver(historicalEnv).GetComponentByName(name) + return configuredRenderUpstreamCommit(historicalEnv, name, allowLockFallback) +} + +func configuredRenderUpstreamCommit( + env *azldev.Env, name string, allowLockFallback bool, +) (commit string, branch string, err error) { + comp, err := components.NewResolver(env).GetComponentByName(name) if err != nil { - return "", "", fmt.Errorf("resolving historical component %#q:\n%w", name, err) + return "", "", fmt.Errorf("resolving component %#q:\n%w", name, err) } commit = comp.GetConfig().Spec.UpstreamCommit if commit == "" && allowLockFallback { - commit, err = lastRenderLockedUpstreamCommit(historicalEnv, name) + commit, err = lastRenderLockedUpstreamCommit(env, name) if err != nil { return "", "", err } @@ -283,9 +295,9 @@ func renderUpstreamCommit( return "", "", fmt.Errorf("component %#q has no configured 'spec.upstream-commit'", name) } - distro, err := sourceproviders.ResolveDistro(historicalEnv, comp) + distro, err := sourceproviders.ResolveDistro(env, comp) if err != nil { - return "", "", fmt.Errorf("resolving historical upstream branch:\n%w", err) + return "", "", fmt.Errorf("resolving upstream branch:\n%w", err) } return commit, distro.Version.DistGitBranch, nil diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index f9a0d5f5c..b8c2ff053 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -93,7 +93,7 @@ dist-git-branch = "f42" return hashes[0] + "\n", nil } - history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl") + history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl", false) require.NoError(t, err) assert.Equal(t, oldSpec, history.spec) require.NotNil(t, history.tree) @@ -139,7 +139,7 @@ func TestReadLastRender_NativeHistoryErrors(t *testing.T) { return testCase.output, testCase.commandErr } - history, err := readLastRender(env.Env, repo, "/project", renderPath, "curl") + history, err := readLastRender(env.Env, repo, "/project", renderPath, "curl", false) if testCase.wantError != "" { require.ErrorContains(t, err, testCase.wantError) } else { @@ -241,7 +241,7 @@ dist-git-branch = "f40" return hashes[0], nil } - history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl") + history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl", false) if testCase.missingCurrent { require.ErrorContains(t, err, "current render upstream commit") @@ -520,7 +520,7 @@ upstream-commit = "bbbbbbb" return hashes[0], nil } - history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl") + history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl", false) if testCase.wantError != "" { require.ErrorContains(t, err, testCase.wantError) From e8c951f0b1a25c20359aa8aa0c7c880f0dabf9b4 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sat, 7 Nov 2026 07:05:00 +0000 Subject: [PATCH 48/70] fix(component): scope lockless render cleanliness checks Limit the lockless render preflight to changes outside the configured project.rendered-specs-dir and changes in selected component dist-git directories. Ignore changes in unrelated rendered output while keeping the --allow-uncommitted override and locked-only behavior unchanged. Use literal Git pathspecs to preserve path boundaries, handle renamed files and inspect the whole repository from nested project directories. Update regression tests, generated help, user docs and agent skills. Validation: mage fix all, mage unit and mage docs passed. Native Git scope checks passed all 34 cases. Quality checks in mage all passed; scenario tests and mage scenarioUpdate remain blocked by the existing Docker-provider/$XDG_RUNTIME_DIR initialization failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 14 ++- .../reference/cli/azldev_component_render.md | 8 +- .../app/azldev/agentskill/agentskill_test.go | 2 + .../azldev/agentskill/content/azldev.md.tmpl | 6 +- .../content/build-component.md.tmpl | 5 +- .../content/refresh-upstream-commit.md.tmpl | 8 +- .../content/withoutlockfile/azldev.md.tmpl | 7 +- .../withoutlockfile/build-component.md.tmpl | 5 +- internal/app/azldev/cmds/component/render.go | 8 +- .../azldev/cmds/component/render_lockless.go | 49 ++++++-- .../component/render_modes_internal_test.go | 115 +++++++++++++++++- 11 files changed, 189 insertions(+), 38 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 95e9380cd..137f69bbe 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -209,17 +209,21 @@ output. Local components with `--no-git-commit` never prompt. All lockless rendering runs on the host without mock; components using lock files retain their existing mock-based rendering. -Before any upstream checkout or component preparation, rendering checks the -entire project Git repository for staged changes, unstaged tracked changes, and -untracked files. Git-ignored files do not block rendering. If any selected -component is lockless, a dirty repository stops the whole invocation. +Before any upstream checkout or component preparation, rendering checks for staged +changes, unstaged tracked changes, and untracked files outside the configured +`project.rendered-specs-dir` and inside every selected component's dist-git +directory (`//`). +Changes elsewhere under `project.rendered-specs-dir`, including other components' +rendered output, are ignored. This scope uses the configured root (default `specs/`), +not a command-line `--output-dir`. Git-ignored files do not block rendering. +If any selected component is lockless, changes in the checked paths stop the whole invocation. Commit or remove those changes first, or explicitly bypass this check: ```sh azldev comp render -p --allow-uncommitted ``` -`--check-only` and `--no-git-commit` still require a clean repository unless +`--check-only` and `--no-git-commit` still require the checked paths to be clean unless `--allow-uncommitted` is specified. The override bypasses only the cleanliness check: carryover and upstream-change comparison still use committed history, and failure recovery can discard uncommitted files in the component output diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 425d5a1f1..2e719f75a 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -69,9 +69,11 @@ preserved. When using a custom output directory (--output-dir), --force is required alongside --clean-stale as a safety measure. This flag is only valid with -a. -Before rendering any lockless component, require a clean Git repository. -Uncommitted tracked changes and untracked files anywhere in the repository block -rendering; Git-ignored files are excluded. --allow-uncommitted bypasses this check. +Before rendering any lockless component, check for uncommitted tracked changes +and untracked files outside 'project.rendered-specs-dir' and in each selected +component's dist-git directory. Other changes under the configured rendered-specs +directory are ignored. Git-ignored files are excluded. +--allow-uncommitted bypasses this check. --check-only and --no-git-commit do not bypass it. Locked components are unaffected. ``` diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 45a173405..cef39a2c8 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -233,6 +233,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.NotContains(t, doc, "host `spectool` filtering") assert.Contains(t, doc, "prepares the changelog before Release") assert.Contains(t, doc, "Git-ignored files are excluded") + assert.Contains(t, doc, "and in selected components' dist-git directories. Other rendered changes are ignored.") assert.Contains(t, doc, "`--allow-uncommitted`") assert.Contains(t, doc, "first commit also includes") assert.Contains(t, doc, "Other rendered paths are excluded even if staged") @@ -270,6 +271,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { require.NoError(t, err) assert.Contains(t, doc, "fully parsed committed TOML") assert.Contains(t, doc, "Git-ignored files are excluded") + assert.Contains(t, doc, "and in selected components' dist-git directories. Other rendered changes are ignored.") assert.Contains(t, doc, "`--allow-uncommitted`") assert.Contains(t, doc, "`render --allow-uncommitted`") assert.Contains(t, doc, "automatically commits that directory") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index c296616c7..f775de02f 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -42,9 +42,9 @@ then runs the refresh pipeline. The list is written in lexical order, one name per line without blank lines or duplicates. Existing names and lock files are retained. Already-listed components are still refreshed; a refresh failure leaves the list updated. -Before preparing any selected lockless component, rendering requires a -clean project Git repository: staged changes, unstaged tracked changes, and -untracked files anywhere in the repository block the whole invocation. +Before preparing any selected lockless component, rendering checks staged changes, +unstaged tracked changes, and untracked files outside `project.rendered-specs-dir` +and in selected components' dist-git directories. Other rendered changes are ignored. Git-ignored files are excluded. Use `--allow-uncommitted` for intentional dirty renders; `--check-only` and `--no-git-commit` do not bypass this check. The override does not change committed-history carryover or failure recovery. diff --git a/internal/app/azldev/agentskill/content/build-component.md.tmpl b/internal/app/azldev/agentskill/content/build-component.md.tmpl index 0e8142eb8..e318bcb88 100644 --- a/internal/app/azldev/agentskill/content/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/build-component.md.tmpl @@ -53,8 +53,9 @@ Prefer `comp render` for quick overlay verification; use `comp diff-sources` to exact overlay effect (it fetches sources once, applies overlays to a copy, and diffs the two trees). Builds can be slow — set generous timeouts. -For all lockless components, uncommitted tracked changes or untracked files -anywhere in the project repository require `comp render --allow-uncommitted`. +For all lockless components, uncommitted tracked changes or untracked files outside +`project.rendered-specs-dir` or in selected components' dist-git directories require +`comp render --allow-uncommitted`. Other rendered changes are ignored. Git-ignored files are excluded; the override does not change committed-history carryover or failure recovery. Commit inputs before the final render (see the `azldev-refresh-upstream-commit` skill). diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 0ef5fc999..0403d3653 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -53,15 +53,15 @@ Commit component inputs before rendering. For upstream components, rendering compares the fully parsed committed TOML at the last render commit and at HEAD; uncommitted generated pins do not count as a committed upstream change. -Before any upstream checkout or component preparation, rendering requires a -clean project Git repository: staged changes, unstaged tracked changes, and -untracked files anywhere in the repository block the whole invocation. +Before any upstream checkout or component preparation, rendering checks staged changes, +unstaged tracked changes, and untracked files outside `project.rendered-specs-dir` +and in selected components' dist-git directories. Other rendered changes are ignored. Git-ignored files are excluded. Use `--allow-uncommitted` for intentional dirty renders; `--check-only` and `--no-git-commit` do not bypass this check. The override does not change committed-history carryover or failure recovery, which can discard uncommitted files in the component output directory. Only locked components do not trigger this check. -Local lockless components also require a clean repository, exclude the TOML defining +Local lockless components use the same scoped cleanliness check, exclude the TOML defining the local spec path, and participate in unchanged-render checks and automatic commits. Their Release and changelog remain unchanged; `--changelog-comment` supplies only their Git message. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 033fd3b04..0bffaefd2 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -39,9 +39,10 @@ The list is written in lexical order, one name per line without blank lines or d Existing names and lock files are retained. Already-listed components are still refreshed; a refresh failure leaves the list updated. -Before preparing any lockless component, rendering requires a clean project Git -repository: staged changes, unstaged tracked changes, and untracked files anywhere -in the repository block the whole invocation. Git-ignored files are excluded. +Before preparing any lockless component, rendering checks staged changes, +unstaged tracked changes, and untracked files outside `project.rendered-specs-dir` +and in selected components' dist-git directories. Other rendered changes are ignored. +Git-ignored files are excluded. Use `--allow-uncommitted` for intentional dirty renders; `--check-only` and `--no-git-commit` do not bypass this check. The override does not change committed-history carryover or failure recovery. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl index dc4b18b8d..3037d9519 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl @@ -56,8 +56,9 @@ two trees). Builds can be slow — set generous timeouts. Finalize with `azldev comp refresh-upstream-commit -p ` before opening a PR (see the `azldev-refresh-upstream-commit` skill). -For all lockless components, uncommitted tracked changes or untracked files anywhere -in the project repository require `comp render --allow-uncommitted`. +For all lockless components, uncommitted tracked changes or untracked files outside +`project.rendered-specs-dir` or in selected components' dist-git directories require +`comp render --allow-uncommitted`. Other rendered changes are ignored. Git-ignored files are excluded; the override does not change committed-history carryover or failure recovery. Commit inputs before the final render. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 4bc676b6d..6f0d0f683 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -50,9 +50,11 @@ func renderOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { parentCmd.AddCommand(NewRenderCmd()) } -const renderWorktreeHelp = `Before rendering any lockless component, require a clean Git repository. -Uncommitted tracked changes and untracked files anywhere in the repository block -rendering; Git-ignored files are excluded. --allow-uncommitted bypasses this check. +const renderWorktreeHelp = `Before rendering any lockless component, check for uncommitted tracked changes +and untracked files outside 'project.rendered-specs-dir' and in each selected +component's dist-git directory. Other changes under the configured rendered-specs +directory are ignored. Git-ignored files are excluded. +--allow-uncommitted bypasses this check. --check-only and --no-git-commit do not bypass it. Locked components are unaffected.` const renderLocalHelp = `Local lockless components copy local files except the TOML defining the spec path. diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index e2455ec8c..4bfa17502 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -84,22 +84,57 @@ func validateRenderComponents( } for _, comp := range comps { - if !env.ComponentWithoutLockfile(comp.GetName()) { - continue + if env.ComponentWithoutLockfile(comp.GetName()) { + return checkRenderWorktree(env, comps) + } + } + + return nil +} + +func checkRenderWorktree(env *azldev.Env, comps []components.Component) error { + specDir := env.Config().Project.RenderedSpecsDir + if specDir == "" { + specDir = projectconfig.DefaultRenderedSpecsDir + } + + if filepath.IsAbs(specDir) { + relative, err := filepath.Rel(env.ProjectDir(), specDir) + if err != nil { + return fmt.Errorf("resolving rendered-specs directory for repository check:\n%w", err) } - status, err := git.RunInDir(env, env, env.ProjectDir(), - "status", "--porcelain=v1", "--untracked-files=all", "--ignore-submodules=none") + specDir = relative + } + + specDir = filepath.Clean(specDir) + componentPaths := make([]string, 0, len(comps)) + + for _, comp := range comps { + name := comp.GetName() + path := filepath.Join(specDir, strings.ToLower(name[:1]), name) + componentPaths = append(componentPaths, ":(literal)"+filepath.ToSlash(path)) + } + + scopes := [][]string{ + {":/", ":(exclude,literal)" + filepath.ToSlash(specDir)}, + componentPaths, + } + for _, scope := range scopes { + args := append([]string{ + "status", "--porcelain=v1", "--untracked-files=all", "--ignore-submodules=none", "--", + }, scope...) + + status, err := git.RunInDir(env, env, env.ProjectDir(), args...) if err != nil { return fmt.Errorf("checking project repository for uncommitted files:\n%w", err) } if status != "" { return errors.New("rendering will not be done due to uncommitted or untracked files " + - "in the project repository; use '--allow-uncommitted' to proceed") + "outside 'project.rendered-specs-dir' or in selected component dist-git directories; " + + "use '--allow-uncommitted' to proceed") } - - return nil } return nil diff --git a/internal/app/azldev/cmds/component/render_modes_internal_test.go b/internal/app/azldev/cmds/component/render_modes_internal_test.go index 3da11203e..34ac01eaa 100644 --- a/internal/app/azldev/cmds/component/render_modes_internal_test.go +++ b/internal/app/azldev/cmds/component/render_modes_internal_test.go @@ -133,12 +133,17 @@ func TestRenderComponentsRequiresCleanLocklessRepository(t *testing.T) { } queried := false env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + scope := []string{":/", ":(exclude,literal)specs"} + if queried { + scope = []string{":(literal)specs/c/curl"} + } + queried = true - assert.Equal(t, []string{ + assert.Equal(t, append([]string{ "git", "-C", "/project", "status", "--porcelain=v1", - "--untracked-files=all", "--ignore-submodules=none", - }, cmd.Args) + "--untracked-files=all", "--ignore-submodules=none", "--", + }, scope...), cmd.Args) return testCase.status, testCase.statusErr } @@ -228,6 +233,96 @@ func TestLocklessRenderWorktreeCheckScope(t *testing.T) { } } +func TestLocklessRenderWorktreeConfiguredScopes(t *testing.T) { + statusErr := errors.New("component status failed") + + for _, testCase := range []struct { + name, specDir, wantRoot string + outsideStatus, componentStatus string + componentError, wantError bool + }{ + {name: "default rendered root"}, + {name: "relative rendered root", specDir: "generated/specs", wantRoot: "generated/specs"}, + {name: "absolute rendered root", specDir: "/project/generated/specs", wantRoot: "generated/specs"}, + {name: "parent rendered root", specDir: "/specs", wantRoot: "../specs"}, + {name: "normalized rendered root", specDir: "generated/../specs/"}, + {name: "literal rendered root", specDir: "generated/[specs]*", wantRoot: "generated/[specs]*"}, + {name: "changes outside rendered root", outsideStatus: " M input.toml\n", wantError: true}, + {name: "selected component modified", componentStatus: " M specs/c/Curl/Curl.spec\n", wantError: true}, + {name: "selected component staged", componentStatus: "M specs/c/Curl/Curl.spec\n", wantError: true}, + {name: "selected component untracked", componentStatus: "?? specs/c/Curl/new.patch\n", wantError: true}, + { + name: "another selected component modified", + componentStatus: " M specs/r/rust-asyncband/rust-asyncband.spec\n", wantError: true, + }, + {name: "selected component status failure", componentError: true, wantError: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.Project.RenderedSpecsDir = testCase.specDir + env.Config.LocklessComponents = map[string]bool{"Curl": true} + ctrl := gomock.NewController(t) + comps := make([]components.Component, 0, 2) + + for _, name := range []string{"Curl", "rust-asyncband"} { + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().AnyTimes().Return(name) + comps = append(comps, comp) + } + + root := testCase.wantRoot + if root == "" { + root = "specs" + } + + calls := 0 + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + calls++ + require.LessOrEqual(t, calls, 2) + + scope := []string{":/", ":(exclude,literal)" + root} + status := testCase.outsideStatus + + if calls == 2 { + scope = []string{":(literal)" + root + "/c/Curl", ":(literal)" + root + "/r/rust-asyncband"} + status = testCase.componentStatus + } + + assert.Equal(t, append([]string{ + "git", "-C", "/project", "status", "--porcelain=v1", + "--untracked-files=all", "--ignore-submodules=none", "--", + }, scope...), cmd.Args) + + if calls == 2 && testCase.componentError { + return "", statusErr + } + + return status, nil + } + + err := validateRenderComponents(env.Env, comps, false) + if testCase.wantError { + require.Error(t, err) + + if testCase.componentError { + require.ErrorIs(t, err, statusErr) + } else { + assert.Contains(t, err.Error(), "--allow-uncommitted") + } + } else { + require.NoError(t, err) + } + + wantCalls := 2 + if testCase.outsideStatus != "" { + wantCalls = 1 + } + + assert.Equal(t, wantCalls, calls) + }) + } +} + func TestLocklessRenderPreservesAllFilesWithoutMock(t *testing.T) { env := testutils.NewTestEnvWithoutLockfile(t) version := env.Config.Distros["test-distro"].Versions["1.0"] @@ -237,11 +332,19 @@ func TestLocklessRenderPreservesAllFilesWithoutMock(t *testing.T) { Name: "curl", Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, } + checked := false env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { - assert.Equal(t, []string{ + scope := []string{":/", ":(exclude,literal)specs"} + if checked { + scope = []string{":(literal)specs/c/curl"} + } + + checked = true + + assert.Equal(t, append([]string{ "git", "-C", "/project", "status", "--porcelain=v1", - "--untracked-files=all", "--ignore-submodules=none", - }, cmd.Args) + "--untracked-files=all", "--ignore-submodules=none", "--", + }, scope...), cmd.Args) return "", nil } From 927ccc7c33532920672ea6de7c9a04b1a9de7194 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 02:29:12 +0000 Subject: [PATCH 49/70] fix(component): preserve locked pins during orphan cleanup Exempt configured locked components from generated upstream-commit orphan checks and pruning, including local components. Keep cleanup for components that have actually been removed. Cover normal refresh and check-only preservation alongside genuine orphan cleanup with in-memory regression tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 2 + .../cmds/component/refreshupstreamcommit.go | 19 ++++++-- .../component/refreshupstreamcommit_test.go | 46 +++++++++++++++++++ internal/upstreamcommit/store.go | 17 +++++-- 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 137f69bbe..04a3f88b1 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -60,6 +60,8 @@ Update (including `--bump`) skips listed components and leaves their locks untouched; refresh skips unlisted components and preserves their generated TOML files. All-component scans still prune files for components that no longer exist, not files belonging to components using the other mode. +This preservation includes generated pins for unlisted local components, both +when refreshing normally and when checking for stale files with `--check-only`. `component history` skips listed components. `component query` extracts their committed rendered dist-git history and processes it with rpmautospec before querying; uncommitted rendered changes are ignored. `component changed` compares resolved project diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit.go b/internal/app/azldev/cmds/component/refreshupstreamcommit.go index be5119847..7c2caca24 100644 --- a/internal/app/azldev/cmds/component/refreshupstreamcommit.go +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit.go @@ -209,7 +209,7 @@ func RefreshUpstreamCommits( // spec-glob-discovered components that aren't in config directly. // Generated TOMLs are version controlled, so pruning is safe even if the // resolved set is empty (e.g., all components removed from config). - wouldPrune, orphanErr := handleOrphanConfigs(store, allComps, options) + wouldPrune, orphanErr := handleOrphanConfigs(env, store, allComps, options) if orphanErr != nil { return filterRefreshDisplayResults(results), orphanErr } @@ -286,12 +286,20 @@ func excludePendingRemovals( // touching disk. Returns (nil, nil) when not running with --all-components, // since orphan handling is scoped to whole-set updates. func handleOrphanConfigs( + env *azldev.Env, store *upstreamcommit.Store, comps []components.Component, options *RefreshUpstreamCommitOptions, ) ([]string, error) { + var ignored []string + for _, comp := range comps { + if !env.ComponentWithoutLockfile(comp.GetName()) { + ignored = append(ignored, comp.GetName()) + } + } + return handleOrphans( - upstreamCommitOrphanStore{store: store}, + upstreamCommitOrphanStore{store: store, ignored: ignored}, comps, options.ComponentFilter.IncludeAllComponents, options.CheckOnly, @@ -304,13 +312,14 @@ func handleOrphanConfigs( // upstreamCommitOrphanStore adapts [upstreamcommit.Store] to [orphanStore]. type upstreamCommitOrphanStore struct { - store *upstreamcommit.Store + store *upstreamcommit.Store + ignored []string } func (s upstreamCommitOrphanStore) findOrphans( resolved map[string]projectconfig.ComponentConfig, ) ([]string, error) { - orphans, err := s.store.FindOrphans(resolved) + orphans, err := s.store.FindOrphans(resolved, s.ignored...) if err != nil { return nil, fmt.Errorf("finding orphan upstream commit TOMLs:\n%w", err) } @@ -321,7 +330,7 @@ func (s upstreamCommitOrphanStore) findOrphans( func (s upstreamCommitOrphanStore) pruneOrphans( resolved map[string]projectconfig.ComponentConfig, ) (int, error) { - pruned, err := s.store.PruneOrphans(resolved) + pruned, err := s.store.PruneOrphans(resolved, s.ignored...) if err != nil { return 0, fmt.Errorf("pruning orphan upstream commit TOMLs:\n%w", err) } diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go index 1bba0d451..b5c5d0740 100644 --- a/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go @@ -194,6 +194,52 @@ func TestRefreshUpstreamCommits_OnlyRefreshesLocklessComponents(t *testing.T) { assert.Equal(t, "preserved-commit", commit) } +func TestRefreshUpstreamCommits_PreservesLockedPinsDuringOrphanCleanup(t *testing.T) { + for _, checkOnly := range []bool{false, true} { + t.Run(map[bool]string{false: "prune", true: "check-only"}[checkOnly], func(t *testing.T) { + env := testutils.NewTestEnv(t) + addRefreshUpstreamComponent(env, "locked-upstream") + env.Config.Components["locked-local"] = projectconfig.ComponentConfig{ + Name: "locked-local", + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: "/project/locked-local.spec", + }, + } + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + for _, name := range []string{"locked-local", "locked-upstream"} { + require.NoError(t, store.Save(name, "preserved-commit")) + } + + options := &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + CheckOnly: checkOnly, + } + results, err := componentcmds.RefreshUpstreamCommits(env.Env, options) + require.NoError(t, err) + assert.Empty(t, results) + for _, name := range []string{"locked-local", "locked-upstream"} { + commit, exists, loadErr := store.Get(name) + require.NoError(t, loadErr) + assert.True(t, exists) + assert.Equal(t, "preserved-commit", commit) + } + + require.NoError(t, store.Save("removed-component", "orphan-commit")) + _, err = componentcmds.RefreshUpstreamCommits(env.Env, options) + if checkOnly { + require.ErrorContains(t, err, "removed-component") + assert.NotContains(t, err.Error(), "locked-local") + } else { + require.NoError(t, err) + } + exists, err := store.Exists("removed-component") + require.NoError(t, err) + assert.Equal(t, checkOnly, exists) + }) + } +} + func TestRefreshUpstreamCommits_ConfigOnlyChangeDoesNotChangeCommitTOML(t *testing.T) { env := testutils.NewTestEnvWithoutLockfile(t) diff --git a/internal/upstreamcommit/store.go b/internal/upstreamcommit/store.go index f0abbb541..907108f4b 100644 --- a/internal/upstreamcommit/store.go +++ b/internal/upstreamcommit/store.go @@ -181,9 +181,11 @@ func (s *Store) Save(componentName, upstreamCommit string) error { } // FindOrphans returns generated component TOMLs that do not correspond to an -// upstream component in components. +// upstream component in components. Configured ignoredComponents are preserved +// regardless of their source type. func (s *Store) FindOrphans( components map[string]projectconfig.ComponentConfig, + ignoredComponents ...string, ) ([]string, error) { entries, err := fileutils.ReadDir(s.fs, s.dir) if err != nil { @@ -199,6 +201,11 @@ func (s *Store) FindOrphans( return nil, fmt.Errorf("reading upstream commit TOML directory %#q:\n%w", s.dir, err) } + ignored := make(map[string]bool, len(ignoredComponents)) + for _, name := range ignoredComponents { + ignored[name] = true + } + var orphans []string for _, entry := range entries { @@ -210,7 +217,7 @@ func (s *Store) FindOrphans( name := strings.TrimSuffix(entry.Name(), fileExtension) component, ok := components[name] - if !ok || component.Spec.SourceType != projectconfig.SpecSourceTypeUpstream { + if !ok || (!ignored[name] && component.Spec.SourceType != projectconfig.SpecSourceTypeUpstream) { orphans = append(orphans, name) } } @@ -221,8 +228,10 @@ func (s *Store) FindOrphans( } // PruneOrphans removes generated TOMLs that are no longer needed. -func (s *Store) PruneOrphans(components map[string]projectconfig.ComponentConfig) (int, error) { - orphans, err := s.FindOrphans(components) +func (s *Store) PruneOrphans( + components map[string]projectconfig.ComponentConfig, ignoredComponents ...string, +) (int, error) { + orphans, err := s.FindOrphans(components, ignoredComponents...) if err != nil { return 0, err } From 92b1388a93f762395452ffafa6e975f2bcb90945 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 02:33:07 +0000 Subject: [PATCH 50/70] fix(git): make single-branch cloning opt-in Restore branch selection without excluding commits on other branches so locked renders and builds retain support for their existing pins. Opt into single-branch fetching explicitly for lockless preparation and snapshot identity resolution. Cover both clone modes and option forwarding. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 2 ++ .../app/azldev/core/sources/sourceprep.go | 4 ++++ .../azldev/core/sources/sourceprep_test.go | 4 +++- .../sourceproviders/fedorasourceprovider.go | 8 ++++++- .../sourceproviders/sourcemanager.go | 6 +++++ internal/utils/git/git.go | 13 ++++++++--- internal/utils/git/git_test.go | 23 ++++++++++++++++++- 7 files changed, 54 insertions(+), 6 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 04a3f88b1..7fdfd708c 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -237,6 +237,8 @@ component's effective resolved distro version's `dist-git-branch` supplies the branch name. The clone retains that branch's full history, not every upstream branch, so pinned commits and snapshot selection remain available without a shallow-history limit. +This branch restriction is specific to lockless preparation. Locked components +retain normal multi-branch cloning so existing pins on other branches remain usable. Immediately after checking out the pinned upstream commit, rendering captures optional Fedora provenance values for `build.emit-upstream-provenance`, before diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index c0af3dc47..d5d1563c9 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -389,6 +389,10 @@ func (p *sourcePreparerImpl) PrepareSources( func (p *sourcePreparerImpl) componentFetchOptions(applyOverlays bool) []sourceproviders.FetchComponentOption { var options []sourceproviders.FetchComponentOption + if p.withoutLockfile { + options = append(options, sourceproviders.WithSingleBranch()) + } + if p.withGitRepo && (applyOverlays || p.skipSyntheticHistory) { options = append(options, sourceproviders.WithPreserveGitDir()) } diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index 7889af784..eaae4a221 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -405,12 +405,14 @@ func TestPrepareSources_WithSkipLookaside_SkipsFetchFiles(t *testing.T) { } assert.True(t, resolved.SkipLookaside, "FetchComponent should receive SkipLookaside option") + assert.True(t, resolved.SingleBranch, "lockless preparation should opt into single-branch cloning") return fileutils.WriteFile(ctx.FS(), outputSpecPath, []byte("# test spec"), fileperms.PublicFile) }, ) - preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx, sources.WithSkipLookaside()) + preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx, + sources.WithSkipLookaside(), sources.WithoutLockfileHistory()) require.NoError(t, err) err = preparer.PrepareSources(ctx, component, testOutputDir, true /*applyOverlays?*/) diff --git a/internal/providers/sourceproviders/fedorasourceprovider.go b/internal/providers/sourceproviders/fedorasourceprovider.go index f6b9b5c76..32d35ec08 100644 --- a/internal/providers/sourceproviders/fedorasourceprovider.go +++ b/internal/providers/sourceproviders/fedorasourceprovider.go @@ -129,12 +129,17 @@ func (g *FedoraSourcesProviderImpl) GetComponent( defer fileutils.RemoveAllAndUpdateErrorIfNil(g.fs, tempDir, &err) // Clone the repository to temp directory with retry for transient network failures. + cloneOptions := []git.GitOptions{git.WithGitBranch(g.distroGitBranch)} + if resolved.SingleBranch { + cloneOptions = append(cloneOptions, git.WithSingleBranch()) + } + err = retry.Do(ctx, g.retryConfig, func() error { // Clean up temp directory contents from any prior failed clone attempt. _ = g.fs.RemoveAll(tempDir) _ = fileutils.MkdirAll(g.fs, tempDir) - return g.gitProvider.Clone(ctx, gitRepoURL, tempDir, git.WithGitBranch(g.distroGitBranch)) + return g.gitProvider.Clone(ctx, gitRepoURL, tempDir, cloneOptions...) }) if err != nil { return fmt.Errorf("failed to clone git repository %#q:\n%w", gitRepoURL, err) @@ -308,6 +313,7 @@ func (g *FedoraSourcesProviderImpl) resolveCommit( return g.gitProvider.Clone(ctx, gitRepoURL, tempDir, git.WithGitBranch(g.distroGitBranch), + git.WithSingleBranch(), git.WithMetadataOnly(), git.WithQuiet(), ) diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index 93b861f7a..53a8577ce 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -77,6 +77,7 @@ type FetchComponentOptions struct { SkipLookaside bool SkipLocalSpecConfig bool + SingleBranch bool } // FetchComponentOption is a functional option for configuring component fetch behavior. @@ -103,6 +104,11 @@ func WithSkipLocalSpecConfig() FetchComponentOption { return func(o *FetchComponentOptions) { o.SkipLocalSpecConfig = true } } +// WithSingleBranch restricts upstream cloning to the selected distro branch. +func WithSingleBranch() FetchComponentOption { + return func(o *FetchComponentOptions) { o.SingleBranch = true } +} + // resolveFetchComponentOptions applies all functional options and returns the resolved options. func resolveFetchComponentOptions(opts []FetchComponentOption) FetchComponentOptions { var resolved FetchComponentOptions diff --git a/internal/utils/git/git.go b/internal/utils/git/git.go index 189ed3aa6..b42998c0a 100644 --- a/internal/utils/git/git.go +++ b/internal/utils/git/git.go @@ -204,11 +204,18 @@ func resolveCloneOptions(options []GitOptions) cloneOptions { return resolved } -// WithGitBranch returns a [GitOptions] that clones only the specified branch -// and its full history. +// WithGitBranch returns a [GitOptions] that selects the initial branch without +// excluding other branches' commits. func WithGitBranch(branch string) GitOptions { return func(opts *cloneOptions) { - opts.args = append(opts.args, "--branch", branch, "--single-branch") + opts.args = append(opts.args, "--branch", branch) + } +} + +// WithSingleBranch returns a [GitOptions] that fetches only the selected branch. +func WithSingleBranch() GitOptions { + return func(opts *cloneOptions) { + opts.args = append(opts.args, "--single-branch") } } diff --git a/internal/utils/git/git_test.go b/internal/utils/git/git_test.go index eac1d21c0..8f49dad3f 100644 --- a/internal/utils/git/git_test.go +++ b/internal/utils/git/git_test.go @@ -126,7 +126,7 @@ func TestCloneWithBranch(t *testing.T) { called = true assert.Equal(t, []string{ - "git", "clone", "--branch", testRepoBranch, "--single-branch", testRepoURL, destDir, + "git", "clone", "--branch", testRepoBranch, testRepoURL, destDir, }, cmd.Args) return nil @@ -143,6 +143,27 @@ func TestCloneWithBranch(t *testing.T) { assert.True(t, called) } +func TestCloneWithSingleBranch(t *testing.T) { + env := testutils.NewTestEnv(t) + provider, err := git.NewGitProviderImpl(env.EventListener, env.CmdFactory) + require.NoError(t, err) + + called := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + called = true + + assert.Equal(t, []string{ + "git", "clone", "--branch", testRepoBranch, "--single-branch", testRepoURL, "/working/repo", + }, cmd.Args) + + return nil + } + + require.NoError(t, provider.Clone(t.Context(), testRepoURL, "/working/repo", + git.WithGitBranch(testRepoBranch), git.WithSingleBranch())) + assert.True(t, called) +} + func TestCloneNonExistentRepo(t *testing.T) { ctrl := gomock.NewController(t) From 05e040df8917648e96f0c09689a08af785b72415 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 02:41:30 +0000 Subject: [PATCH 51/70] fix(component): resolve autorelease before static transitions Calculate the previous complete release from committed component history instead of restoring its autorelease macro into a new static spec. Resolve its generated changelog before the normal static bump as well. Preserve version-reset and no-dist behavior, propagate calculation errors, and remove temporary repositories on both success and failure. Cover both spec editors and document the extraction prerequisite. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 8 ++ .../cmds/component/refreshupstreamcommit.go | 1 + .../component/refreshupstreamcommit_test.go | 4 + .../azldev/cmds/component/render_lockless.go | 6 ++ .../render_lockless_internal_test.go | 6 ++ .../component/render_release_carryover.go | 95 +++++++++++++++- .../render_release_carryover_internal_test.go | 101 ++++++++++++++++++ 7 files changed, 219 insertions(+), 2 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 7fdfd708c..9fbc10b4e 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -281,6 +281,14 @@ unchanged. A missing or ambiguous macro definition fails rendering. Otherwise, it copies the committed `Release:` value into the upstream spec's `Release:`. No temporary release-bump probe is performed. +When upstream switches from `%autorelease` to a static release, the renderer +first extracts the committed component history and calculates its complete +release with host rpmautospec. For an unchanged version, that concrete release +is the baseline for the normal static bump; the old `%autorelease` macro is +never restored. An existing `%autochangelog` is materialized from the same +history before adding the static changelog entry. This transition requires +`git-filter-repo`; a changed version still starts from the new upstream release. + For an initial `%autorelease` render, it calculates the numeric upstream release and adds or replaces the macro's `-b` parameter. This rpmautospec operation runs on the host; it does not preprocess or rewrite the spec. Initial and legacy diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit.go b/internal/app/azldev/cmds/component/refreshupstreamcommit.go index 7c2caca24..6e99dc46d 100644 --- a/internal/app/azldev/cmds/component/refreshupstreamcommit.go +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit.go @@ -292,6 +292,7 @@ func handleOrphanConfigs( options *RefreshUpstreamCommitOptions, ) ([]string, error) { var ignored []string + for _, comp := range comps { if !env.ComponentWithoutLockfile(comp.GetName()) { ignored = append(ignored, comp.GetName()) diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go index b5c5d0740..5c8be0ad6 100644 --- a/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go @@ -206,6 +206,7 @@ func TestRefreshUpstreamCommits_PreservesLockedPinsDuringOrphanCleanup(t *testin Path: "/project/locked-local.spec", }, } + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) for _, name := range []string{"locked-local", "locked-upstream"} { require.NoError(t, store.Save(name, "preserved-commit")) @@ -218,6 +219,7 @@ func TestRefreshUpstreamCommits_PreservesLockedPinsDuringOrphanCleanup(t *testin results, err := componentcmds.RefreshUpstreamCommits(env.Env, options) require.NoError(t, err) assert.Empty(t, results) + for _, name := range []string{"locked-local", "locked-upstream"} { commit, exists, loadErr := store.Get(name) require.NoError(t, loadErr) @@ -226,6 +228,7 @@ func TestRefreshUpstreamCommits_PreservesLockedPinsDuringOrphanCleanup(t *testin } require.NoError(t, store.Save("removed-component", "orphan-commit")) + _, err = componentcmds.RefreshUpstreamCommits(env.Env, options) if checkOnly { require.ErrorContains(t, err, "removed-component") @@ -233,6 +236,7 @@ func TestRefreshUpstreamCommits_PreservesLockedPinsDuringOrphanCleanup(t *testin } else { require.NoError(t, err) } + exists, err := store.Exists("removed-component") require.NoError(t, err) assert.Equal(t, checkOnly, exists) diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 4bfa17502..fa2d136ad 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -65,6 +65,7 @@ type lastRender struct { type renderAutorelease interface { GenerateChangelog(ctx context.Context, dir, filename string) (string, error) CalculateBaseRelease(ctx context.Context, dir, filename string) (string, error) + CalculateRelease(ctx context.Context, dir, filename string) (string, error) ExpandRelease(ctx context.Context, dir, filename string) (string, error) } @@ -410,6 +411,11 @@ func prepareLocklessSpec( return err } + history, err = staticRenderHistory(env, autospec, dir, name, original, history, editor) + if err != nil { + return err + } + if err := prepareLocklessChangelog(env, autospec, dir, filepath.Base(path), original, history); err != nil { return err } diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index b8c2ff053..d21b56b88 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -557,6 +557,12 @@ func (s *renderAutospecStub) CalculateBaseRelease(_ context.Context, _, _ string return "Calculated release number: 42\n", s.err } +func (s *renderAutospecStub) CalculateRelease(_ context.Context, dir, filename string) (string, error) { + s.calls = append(s.calls, "complete-release") + + return s.expand(dir, filename) +} + func (s *renderAutospecStub) ExpandRelease(_ context.Context, dir, filename string) (string, error) { s.calls = append(s.calls, "expand") diff --git a/internal/app/azldev/cmds/component/render_release_carryover.go b/internal/app/azldev/cmds/component/render_release_carryover.go index ba257e61c..0d05691fa 100644 --- a/internal/app/azldev/cmds/component/render_release_carryover.go +++ b/internal/app/azldev/cmds/component/render_release_carryover.go @@ -4,20 +4,111 @@ package component import ( + "errors" "fmt" + "path/filepath" "regexp" "strings" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" ) var ( - baseReleaseMacro = regexp.MustCompile(`%(?:\{\??baserelease\}|baserelease\b)`) - baseReleaseLine = regexp.MustCompile(`^[ \t]*%(?:global|define)[ \t]+baserelease[ \t]+\S`) + baseReleaseMacro = regexp.MustCompile(`%(?:\{\??baserelease\}|baserelease\b)`) + baseReleaseLine = regexp.MustCompile(`^[ \t]*%(?:global|define)[ \t]+baserelease[ \t]+\S`) + autoChangelogMacro = regexp.MustCompile(`%(?:\{\??autochangelog\}|autochangelog\b)`) + autoreleaseNoDist = regexp.MustCompile(`(?:^|\s)-n(?:\s|\}|$)`) ) +func staticRenderHistory( + env *azldev.Env, autospec renderAutorelease, dir, name, release string, history lastRender, editor spec.EditorMode, +) (resolved lastRender, err error) { + if history.spec == "" || env.DryRun() || sources.ReleaseUsesAutorelease(release) { + return history, nil + } + + previous, err := spec.OpenSpec(strings.NewReader(history.spec), spec.WithEditor(editor)) + if err != nil { + return history, fmt.Errorf("parsing committed release spec:\n%w", err) + } + + previousRelease, err := previous.GetLastTag("", "Release") + if err != nil { + return history, fmt.Errorf("reading committed Release:\n%w", err) + } + + if !sources.ReleaseUsesAutorelease(previousRelease) { + return history, nil + } + + scratch, err := fileutils.MkdirTemp(env.FS(), filepath.Dir(dir), "azldev-render-static-") + if err != nil { + return history, fmt.Errorf("creating committed autorelease scratch directory:\n%w", err) + } + defer fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), scratch, &err) + + snapshot := filepath.Join(scratch, "dist-git") + if _, err := CreateComponentDistGitRepo(env, &DistGitRepoOptions{ + Component: name, OutputDir: snapshot, + }); err != nil { + return history, fmt.Errorf("extracting committed autorelease history:\n%w", err) + } + + filename := name + ".spec" + + release, err = autospec.CalculateRelease(env, snapshot, filename) + if err != nil { + return history, fmt.Errorf("calculating committed autorelease:\n%w", err) + } + + fields := strings.Fields(release) + if len(fields) == 0 { + return history, errors.New("calculating committed autorelease: empty release") + } + + release = fields[len(fields)-1] + if !autoreleaseNoDist.MatchString(previousRelease) { + release += "%{?dist}" + } + + if err := previous.SetTag("", "Release", release); err != nil { + return history, fmt.Errorf("carrying calculated autorelease:\n%w", err) + } + + var content strings.Builder + if err := previous.Serialize(&content); err != nil { + return history, fmt.Errorf("serializing committed release spec:\n%w", err) + } + + resolved = history + resolved.spec, err = staticRenderChangelog(env, autospec, snapshot, filename, content.String(), editor) + + return resolved, err +} + +func staticRenderChangelog( + env *azldev.Env, autospec renderAutorelease, dir, filename, content string, editor spec.EditorMode, +) (string, error) { + if !autoChangelogMacro.MatchString(content) { + return content, nil + } + + changelog, err := autospec.GenerateChangelog(env, dir, filename) + if err != nil { + return "", fmt.Errorf("generating committed autorelease changelog:\n%w", err) + } + + start, end, err := renderChangelogBounds([]byte(content), editor) + if err != nil { + return "", err + } + + return content[:start] + "%changelog\n" + changelog + content[end:], nil +} + func carryStaticRenderRelease( env *azldev.Env, working *spec.Spec, release string, history lastRender, editor spec.EditorMode, diff --git a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go index 90fb9f34d..b4a0b8510 100644 --- a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go +++ b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go @@ -4,11 +4,14 @@ package component import ( + "errors" "os/exec" + "path/filepath" "strings" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" @@ -16,6 +19,104 @@ import ( "github.com/stretchr/testify/require" ) +func TestPrepareLocklessSpec_AutoreleaseToStatic(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + for _, testCase := range []struct { + name, version, baseline, previous string + fail bool + }{ + {name: "same version", version: "1", baseline: "12%{?dist}"}, + {name: "new version", version: "2", baseline: "1%{?dist}"}, + {name: "without dist", version: "1", baseline: "12", previous: "%{autorelease -b 8 -n}"}, + {name: "release query fails", version: "1", fail: true}, + } { + t.Run(string(editor)+"/"+testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.Project.RenderedSpecsDir = "/project/specs" + env.Config.Components["curl"] = projectconfig.ComponentConfig{Name: "curl"} + env.CmdFactory.RegisterCommandInSearchPath("git") + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + previous := testCase.previous + if previous == "" { + previous = "%autorelease -b 8" + } + + history := lastRender{spec: renderTestSpec("1", previous, "%autochangelog\n")} + + const path = "/working/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, path, + []byte(renderTestSpec(testCase.version, "1%{?dist}", "* Upstream entry\n")), + fileperms.PublicFile)) + + var snapshot string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + switch cmd.Args[3] { + case renderTestRevParse: + if cmd.Args[4] == "--git-common-dir" { + return "/project/.git", nil + } + + return renderTestProjectRoot, nil + case "clone": + snapshot = cmd.Args[len(cmd.Args)-1] + require.NoError(t, fileutils.MkdirAll(env.TestFS, filepath.Join(snapshot, ".git"))) + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(snapshot, "curl.spec"), + []byte(history.spec), fileperms.PublicFile)) + } + + return "", nil + } + queryErr := errors.New("release query failed") + autospec := &renderAutospecStub{expand: func(dir, filename string) (string, error) { + assert.Equal(t, snapshot, dir) + assert.Equal(t, "curl.spec", filename) + content, err := fileutils.ReadFile(env.TestFS, filepath.Join(dir, filename)) + require.NoError(t, err) + assert.Equal(t, history.spec, string(content)) + + if testCase.fail { + return "", queryErr + } + + return "Calculated release number: 12\n", nil + }} + + err := prepareLocklessSpec(env.Env, autospec, "/working", "curl", editor, history) + if testCase.fail { + require.ErrorIs(t, err, queryErr) + } else { + require.NoError(t, err) + + content, readErr := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, readErr) + assert.Contains(t, string(content), "Release: "+testCase.baseline+"\n") + assert.Contains(t, string(content), "* Full upstream history\n") + assert.NotContains(t, string(content), "%autorelease") + assert.NotContains(t, string(content), "%autochangelog") + + bumped := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + bumped = true + + assert.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) + + return nil + } + _, err = bumpLocklessSpec(env.Env, path, &locklessRenderUpdate{subject: "test transition"}) + require.NoError(t, err) + assert.True(t, bumped) + } + + exists, checkErr := fileutils.Exists(env.TestFS, snapshot) + require.NoError(t, checkErr) + assert.False(t, exists, "remove the temporary history on success and failure") + }) + } + } +} + func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { t.Run(string(editor), func(t *testing.T) { From 893ef1668ff3546900e257a44bfa4a56cb63ff1f Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 02:58:55 +0000 Subject: [PATCH 52/70] perf(component): share project objects for lockless queries Clone one immutable project snapshot for each bulk query, then extract component histories concurrently using shared objects instead of repeating full project clones serially. Repack each filtered repository and remove its alternate object dependency before passing it into Mock. Cover bulk and mixed queries, extraction failures, independent object stores, and scratch cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 10 +- .../azldev/cmds/component/query_lockless.go | 107 +++++++++- .../app/azldev/cmds/component/query_test.go | 194 +++++++++++++----- 3 files changed, 249 insertions(+), 62 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 9fbc10b4e..b7c4b1823 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -28,8 +28,8 @@ rust-asyncband Rendering, builds, and source preparation ignore existing locks and lock validation for listed components. They use configured upstream commits. -Lockless rendering does not generate synthetic history; builds and source -preparation still derive it from committed upstream-commit TOML changes. +Lockless rendering, builds, queries, and source preparation do not generate +synthetic history; builds and queries use committed rendered history. Unlisted components use lock files, lock validation, and fingerprint-derived synthetic history. Both kinds can be processed together. @@ -90,6 +90,12 @@ using the component's repository-relative rendered directory. Install `git-filter-repo` on the host. Uncommitted rendered changes are not included, and extraction never modifies the original repository. +For a bulk query, the project branch is cloned once into a temporary bare +snapshot. Component repositories initially share that immutable object store +and are filtered with bounded CPU concurrency. Each filtered repository is then +repacked and detached from the shared store before entering Mock, so it contains +only its own history and does not depend on host-only object paths. + `component query` processes the temporary spec with rpmautospec in the project's Mock chroot, then reports its source NEVR, required files, and binary subpackages. The temporary query repositories are removed on success and failure. diff --git a/internal/app/azldev/cmds/component/query_lockless.go b/internal/app/azldev/cmds/component/query_lockless.go index 4546200a6..f4ff11d2f 100644 --- a/internal/app/azldev/cmds/component/query_lockless.go +++ b/internal/app/azldev/cmds/component/query_lockless.go @@ -4,6 +4,7 @@ package component import ( + "context" "errors" "fmt" "path/filepath" @@ -12,6 +13,8 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/parmap" ) func splitQueryComponents( @@ -54,17 +57,15 @@ func queryLocklessComponents( env *azldev.Env, processor *sources.MockProcessor, comps []components.Component, scratchDir, arch string, ) ([]sources.SpecQueryResult, error) { stagingDir := filepath.Join(scratchDir, "dist-git") + if err := extractQueryDistGitRepos(env, comps, stagingDir); err != nil { + return nil, err + } + processInputs := make([]sources.ComponentInput, 0, len(comps)) queryInputs := make([]sources.SpecQueryInput, 0, len(comps)) for _, comp := range comps { name := comp.GetName() - if _, err := CreateComponentDistGitRepo(env, &DistGitRepoOptions{ - Component: name, OutputDir: filepath.Join(stagingDir, name), - }); err != nil { - return nil, fmt.Errorf("extracting dist-git for %#q:\n%w", name, err) - } - specFilename := name + ".spec" processInputs = append(processInputs, sources.ComponentInput{ Name: name, SpecFilename: specFilename, SkipSpectool: true, @@ -103,3 +104,97 @@ func queryLocklessComponents( return append(results, failures...), nil } + +func extractQueryDistGitRepos(env *azldev.Env, comps []components.Component, stagingDir string) error { + if env.DryRun() { + return nil + } + + repoRoot, err := git.RunInDir(env, env, env.ProjectDir(), "rev-parse", "--show-toplevel") + if err != nil { + return fmt.Errorf("finding query project repository:\n%w", err) + } + + if err := validateDistGitRepoOutput(env, repoRoot, stagingDir); err != nil { + return err + } + + if err := fileutils.MkdirAll(env.FS(), stagingDir); err != nil { + return fmt.Errorf("creating query dist-git directory:\n%w", err) + } + + source := filepath.Join(filepath.Dir(stagingDir), "project.git") + if _, err := git.RunInDir(env, env, repoRoot, + "clone", "--single-branch", "--no-local", "--bare", "--", repoRoot, source); err != nil { + return fmt.Errorf("cloning query project snapshot:\n%w", err) + } + + workerEnv, cancel := env.WithCancel() + defer cancel() + + results := parmap.Map(workerEnv, env.CPUBoundConcurrency(), comps, nil, + func(ctx context.Context, comp components.Component) error { + return extractQueryComponent(ctx, workerEnv, repoRoot, source, stagingDir, comp) + }) + + var failures []error + + for idx, result := range results { + extractErr := result.Value + if result.Cancelled { + extractErr = workerEnv.Err() + } + + if extractErr != nil { + failures = append(failures, fmt.Errorf("extracting dist-git for %#q:\n%w", + comps[idx].GetName(), extractErr)) + } + } + + return errors.Join(failures...) +} + +func extractQueryComponent( + ctx context.Context, env *azldev.Env, repoRoot, source, stagingDir string, comp components.Component, +) error { + name := comp.GetName() + if err := fileutils.ValidateFilename(name); err != nil { + return fmt.Errorf("invalid query component name:\n%w", err) + } + + subdir, err := distGitRepoSubdirectory(env, repoRoot, name) + if err != nil { + return err + } + + output := filepath.Join(stagingDir, name) + if _, err := git.RunInDir(ctx, env, repoRoot, + "clone", "--shared", "--no-checkout", "--single-branch", "--", source, output); err != nil { + return fmt.Errorf("cloning shared query history:\n%w", err) + } + + if _, err := git.RunInDir(ctx, env, output, + "filter-repo", "--force", "--subdirectory-filter", subdir); err != nil { + return fmt.Errorf("filtering query component history:\n%w", err) + } + + // Copy only the filtered component's reachable objects before moving it into Mock. + if _, err := git.RunInDir(ctx, env, output, "repack", "-a", "-d"); err != nil { + return fmt.Errorf("packing query component history:\n%w", err) + } + + alternates := filepath.Join(output, ".git", "objects", "info", "alternates") + + exists, err := fileutils.Exists(env.FS(), alternates) + if err != nil { + return fmt.Errorf("checking shared query objects:\n%w", err) + } + + if exists { + if err := env.FS().Remove(alternates); err != nil { + return fmt.Errorf("detaching shared query objects:\n%w", err) + } + } + + return nil +} diff --git a/internal/app/azldev/cmds/component/query_test.go b/internal/app/azldev/cmds/component/query_test.go index eb388ba15..531629c36 100644 --- a/internal/app/azldev/cmds/component/query_test.go +++ b/internal/app/azldev/cmds/component/query_test.go @@ -87,22 +87,118 @@ func TestNewComponentQueryCommand(t *testing.T) { assert.Equal(t, "query", cmd.Use) } +type queryDistGitFixture struct { + cloneDir string + filtered bool + fullClones, sharedClones, repacked int +} + +func mockQueryDistGit( + t *testing.T, env *testutils.TestEnv, failure string, sentinel error, +) *queryDistGitFixture { + t.Helper() + + fixture := &queryDistGitFixture{} + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[0] != "git" { + return "", nil + } + + switch cmd.Args[3] { + case "rev-parse": + if cmd.Args[4] == "--git-common-dir" { + return distGitTestMetadata, nil + } + + return "/project", nil + case distGitTestClone: + if failure == distGitTestClone { + return "", sentinel + } + + if slices.Contains(cmd.Args, "--bare") { + fixture.fullClones++ + + assert.Equal(t, []string{"--single-branch", "--no-local", "--bare", "--"}, cmd.Args[4:8]) + + return "", nil + } + + fixture.sharedClones++ + + assert.Equal(t, []string{"--shared", "--no-checkout", "--single-branch", "--"}, cmd.Args[4:8]) + fixture.cloneDir = cmd.Args[len(cmd.Args)-1] + assert.Equal(t, filepath.Join(filepath.Dir(filepath.Dir(fixture.cloneDir)), "project.git"), + cmd.Args[len(cmd.Args)-2]) + require.NoError(t, fileutils.WriteFile(env.FS(), + filepath.Join(fixture.cloneDir, ".git", "objects", "info", "alternates"), + []byte("shared objects"), fileperms.PublicFile)) + case distGitTestFilterRepo: + if failure == distGitTestFilterRepo { + return "", sentinel + } + + if slices.Contains(cmd.Args, "--subdirectory-filter") { + if failure == "filter" { + return "", sentinel + } + + name := filepath.Base(cmd.Args[2]) + assert.Equal(t, "specs/"+strings.ToLower(name[:1])+"/"+name, cmd.Args[len(cmd.Args)-1]) + + fixture.filtered = true + + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(cmd.Args[2], name+".spec"), + []byte("committed spec"), fileperms.PublicFile)) + } + case "repack": + if failure == "repack" { + return "", sentinel + } + + fixture.repacked++ + + assert.Equal(t, []string{"-a", "-d"}, cmd.Args[4:]) + } + + return "", nil + } + + return fixture +} + func TestQueryComponents_LocklessCommittedDistGit(t *testing.T) { for _, testCase := range []struct { name, failure string mixed bool + multiple bool }{ {name: "lockless"}, {name: "mixed", mixed: true}, + {name: "multiple lockless components", multiple: true}, {name: "missing filter-repo", failure: distGitTestFilterRepo}, + {name: "snapshot clone failure", failure: distGitTestClone}, + {name: "filter failure", failure: "filter"}, + {name: "repack failure", failure: "repack"}, {name: "processing failure", failure: "process"}, {name: "query failure", failure: "query"}, } { t.Run(testCase.name, func(t *testing.T) { failure := testCase.failure env := newDistGitRepoTestEnv(t, testComponentName) + env.Env.SetConcurrency(1) + + names := []string{testComponentName} + if testCase.multiple { + names = append(names, "rust-asyncband") + env.Config.Components["rust-asyncband"] = projectconfig.ComponentConfig{Name: "rust-asyncband"} + } + + env.Config.LocklessComponents = make(map[string]bool, len(names)) + for _, name := range names { + env.Config.LocklessComponents[name] = true + } - env.Config.LocklessComponents = map[string]bool{testComponentName: true} if testCase.mixed { env.Config.Components["bash"] = projectconfig.ComponentConfig{ Name: "bash", @@ -116,47 +212,10 @@ func TestQueryComponents_LocklessCommittedDistGit(t *testing.T) { sentinel := errors.New("injected failure") - var ( - cloneDir string - cloned, filtered, processed, queried bool - ) - - env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { - if cmd.Args[0] != "git" { - return "", nil - } - - switch cmd.Args[3] { - case "rev-parse": - if cmd.Args[4] == "--git-common-dir" { - return distGitTestMetadata, nil - } - - return "/project", nil - case distGitTestClone: - assert.Equal(t, []string{"--single-branch", "--no-local", "--"}, - cmd.Args[4:7]) - cloneDir = cmd.Args[len(cmd.Args)-1] - cloned = true - - require.NoError(t, fileutils.MkdirAll(env.FS(), cloneDir)) - case distGitTestFilterRepo: - if failure == distGitTestFilterRepo { - return "", sentinel - } - - if slices.Contains(cmd.Args, "--subdirectory-filter") { - assert.Equal(t, "specs/c/curl", cmd.Args[len(cmd.Args)-1]) + fixture := mockQueryDistGit(t, env, failure, sentinel) - filtered = true + var processed, queried bool - require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(cloneDir, "curl.spec"), - []byte("committed spec"), fileperms.PublicFile)) - } - } - - return "", nil - } env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { args := strings.Join(cmd.Args, " ") if !strings.Contains(args, "--chroot") { @@ -170,18 +229,32 @@ func TestQueryComponents_LocklessCommittedDistGit(t *testing.T) { processed = true - assert.True(t, filtered) + assert.True(t, fixture.filtered) + + var records []map[string]any + + for _, name := range names { + dir := filepath.Join(filepath.Dir(fixture.cloneDir), name) + content, err := fileutils.ReadFile(env.FS(), filepath.Join(dir, name+".spec")) + require.NoError(t, err) + assert.Equal(t, "committed spec", string(content)) - content, err := fileutils.ReadFile(env.FS(), filepath.Join(cloneDir, "curl.spec")) + exists, err := fileutils.Exists(env.FS(), filepath.Join(dir, ".git", "objects", "info", "alternates")) + require.NoError(t, err) + assert.False(t, exists, "component repositories must not depend on host-only shared paths") + + records = append(records, map[string]any{"name": name, "specFiles": "", "error": nil}) + } + + data, err := json.Marshal(records) require.NoError(t, err) - assert.Equal(t, "committed spec", string(content)) - return fileutils.WriteFile(env.FS(), filepath.Join(filepath.Dir(cloneDir), "results.json"), - []byte(`[{"name":"curl","specFiles":"","error":null}]`), fileperms.PublicFile) + return fileutils.WriteFile(env.FS(), filepath.Join(filepath.Dir(fixture.cloneDir), "results.json"), + data, fileperms.PublicFile) } if strings.Contains(args, "query_process.py") { - if testCase.mixed && !filtered { + if testCase.mixed && !fixture.filtered { return lockedQueryHandler(cmd) } @@ -193,15 +266,25 @@ func TestQueryComponents_LocklessCommittedDistGit(t *testing.T) { assert.True(t, processed) - return fileutils.WriteFile(env.FS(), filepath.Join(filepath.Dir(filepath.Dir(cloneDir)), "results.json"), - []byte(`[{"name":"curl","srpmOut":"name=curl\nepoch=0\nversion=1\nrelease=3\n","binOut":"","error":null}]`), - fileperms.PublicFile) + var records []map[string]any + for _, name := range names { + records = append(records, map[string]any{ + "name": name, "srpmOut": "name=" + name + "\nepoch=0\nversion=1\nrelease=3\n", + "binOut": "", "error": nil, + }) + } + + data, err := json.Marshal(records) + require.NoError(t, err) + + return fileutils.WriteFile(env.FS(), filepath.Join(filepath.Dir(filepath.Dir(fixture.cloneDir)), "results.json"), + data, fileperms.PublicFile) } return nil } - filter := components.ComponentFilter{ComponentNamePatterns: []string{testComponentName}} + filter := components.ComponentFilter{ComponentNamePatterns: names} if testCase.mixed { filter.IncludeAllComponents = true } @@ -225,15 +308,18 @@ func TestQueryComponents_LocklessCommittedDistGit(t *testing.T) { require.NoError(t, queryErr) - wantLen := 1 + wantLen := len(names) if testCase.mixed { - wantLen = 2 + wantLen++ } require.Len(t, results, wantLen) - assert.True(t, cloned && filtered && processed && queried) + assert.True(t, fixture.filtered && processed && queried) + assert.Equal(t, 1, fixture.fullClones, "clone project objects only once for the whole query") + assert.Equal(t, len(names), fixture.sharedClones) + assert.Equal(t, len(names), fixture.repacked) - releases := map[string]string{testComponentName: "3", "bash": "1.azl4"} + releases := map[string]string{testComponentName: "3", "rust-asyncband": "3", "bash": "1.azl4"} for _, result := range results { assert.Equal(t, releases[result.Name], result.Version.Release()) } From e3400055323ef11d77b8380355d08c14cced75ca Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 03:26:06 +0000 Subject: [PATCH 53/70] fix(component): skip lockless source preparation during dry runs The lockless prepare-sources path called CheckOutputDir before checking for dry-run mode. With '--force', that helper removes a nonempty output directory through the non-dry-run-aware filesystem, so a dry run could delete existing output. During a dry run, still reject a nonempty output directory when '--force' is not set, but return before any cleanup or preparation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ae64263c-02b0-4e70-89b2-030ec0389a00 --- .../cmds/component/preparesources_lockless.go | 15 ++++++++ .../preparesources_lockless_internal_test.go | 38 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/internal/app/azldev/cmds/component/preparesources_lockless.go b/internal/app/azldev/cmds/component/preparesources_lockless.go index 08b14beef..b8f9cd897 100644 --- a/internal/app/azldev/cmds/component/preparesources_lockless.go +++ b/internal/app/azldev/cmds/component/preparesources_lockless.go @@ -6,6 +6,7 @@ package component import ( "context" "fmt" + "log/slog" "path/filepath" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" @@ -42,6 +43,20 @@ func prepareLocklessSources( return fmt.Errorf("creating source manager for %#q:\n%w", component.GetName(), err) } + if env.DryRun() { + // [CheckOutputDir] only mutates the filesystem when '--force' is set. + if !options.Force { + if err := CheckOutputDir(env, options); err != nil { + return err + } + } + + slog.Info("Dry run; would prepare lockless sources", + "component", component.GetName(), "outputDir", options.OutputDir) + + return nil + } + if err := CheckOutputDir(env, options); err != nil { return err } diff --git a/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go b/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go index e8e0004ef..dbd9ccf3b 100644 --- a/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go @@ -273,6 +273,44 @@ func TestLocklessPrepareSourcesRejectsNonemptyOutputWithoutForce(t *testing.T) { assert.Equal(t, "original output", string(content)) } +func TestLocklessPrepareSourcesDryRunPreservesForcedOutput(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + } + dryEnv := azldev.NewEnv(t.Context(), azldev.EnvOptions{ + ProjectDir: env.Env.ProjectDir(), Config: env.Config, Interfaces: env.TestInterfaces, + EventListener: env.EventListener, DryRunnable: azldev.NewAppDryRunnable(true), + }) + + const output = "/external/output" + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(output, "keep"), + []byte("original output"), fileperms.PublicFile)) + + originalPreparer := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, _ opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + t.Fatal("dry run must not prepare sources") + + return nil, errors.New("unexpected source preparer construction") + } + + t.Cleanup(func() { newSourcePreparer = originalPreparer }) + + require.NoError(t, PrepareComponentSources(dryEnv, &PrepareSourcesOptions{ + ComponentFilter: components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}, + OutputDir: output, + Force: true, + })) + + content, err := fileutils.ReadFile(env.FS(), filepath.Join(output, "keep")) + require.NoError(t, err) + assert.Equal(t, "original output", string(content)) +} + func TestLocklessPrepareSourcesUsesWorkingInputs(t *testing.T) { const fetchCommand = "fetch" From 44dde8a5a5c374df866734ec365eda9bbd380df5 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 03:28:23 +0000 Subject: [PATCH 54/70] fix(component): preserve locked render alias collisions Restore non-symlink collision protection for locked render aliases while retaining replacement of occupied alias paths for lockless components. Cover file and directory collisions in both modes and retain the existing relative and dangling-symlink regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 2 + internal/app/azldev/cmds/component/render.go | 28 +++++++-- .../cmds/component/render_internal_test.go | 59 +++++++++++++++++-- 3 files changed, 80 insertions(+), 9 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index b7c4b1823..1bcf582f0 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -32,6 +32,8 @@ Lockless rendering, builds, queries, and source preparation do not generate synthetic history; builds and queries use committed rendered history. Unlisted components use lock files, lock validation, and fingerprint-derived synthetic history. Both kinds can be processed together. +Locked rendering preserves existing non-symlink entries at percent-encoded alias +paths and warns about the collision; lockless rendering replaces those entries. A missing or empty file means all components use lock files. Blank lines and surrounding whitespace are ignored; names are not glob patterns. A file read diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 6f0d0f683..e1d04bfef 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "log/slog" + "os" "path/filepath" "slices" "sort" @@ -929,7 +930,7 @@ func finishComponentRender( func writeRenderedAlias(env *azldev.Env, prep *preparedComponent) { name := prep.comp.GetName() - if err := writeAliasSymlink(env.FS(), prep.compOutputDir, name); err != nil { + if err := writeAliasSymlink(env.FS(), prep.compOutputDir, name, prep.lockless); err != nil { slog.Warn("Failed to create rendered-spec alias symlink; downstream build steps"+ " that consume the percent-encoded path may fail to locate this component", "component", name, "error", err) @@ -943,9 +944,8 @@ func writeRenderedAlias(env *azldev.Env, prep *preparedComponent) { // No-ops when no encoding is needed (plain ASCII names) or when the underlying // filesystem doesn't support symlinks (e.g., in-memory test FS). // -// Removes any existing entry at the alias path, including nonempty directories, -// before creating the symlink. -func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName string) error { +// Only lockless rendering replaces non-symlink entries at the alias path. +func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName string, replaceExisting bool) error { aliasName := components.RenderedSpecDirAliasName(componentName) if aliasName == "" { return nil @@ -962,6 +962,16 @@ func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName st parentDir := filepath.Dir(componentOutputDir) aliasPath := filepath.Join(parentDir, aliasName) + if !replaceExisting { + info, err := lstatIfPossible(fileSystem, aliasPath) + switch { + case err == nil && info.Mode()&os.ModeSymlink == 0: + return fmt.Errorf("alias path %#q is already occupied by a non-symlink entry; refusing to overwrite", aliasPath) + case err != nil && !errors.Is(err, os.ErrNotExist): + return fmt.Errorf("inspecting alias path %#q:\n%w", aliasPath, err) + } + } + if removeErr := fileSystem.RemoveAll(aliasPath); removeErr != nil { return fmt.Errorf("removing existing alias path %#q:\n%w", aliasPath, removeErr) } @@ -975,6 +985,16 @@ func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName st return nil } +func lstatIfPossible(fileSystem opctx.FS, path string) (os.FileInfo, error) { + if lstater, ok := fileSystem.(afero.Lstater); ok { + info, _, err := lstater.LstatIfPossible(path) + + return info, err //nolint:wrapcheck // Preserve the filesystem error. + } + + return fileSystem.Stat(path) //nolint:wrapcheck // Preserve the filesystem error. +} + // moveRenderedOutput recreates the output directory and moves all prepared entries // except '.git' into it. Existing output requires allowOverwrite. func moveRenderedOutput(env *azldev.Env, tempDir string, prep *preparedComponent, allowOverwrite bool) error { diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index e77de67bc..0df271f66 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -5,6 +5,7 @@ package component import ( "errors" + "fmt" "os" "os/exec" "path/filepath" @@ -439,7 +440,7 @@ func TestWriteAliasSymlink(t *testing.T) { realDir := filepath.Join(letterDir, "libxml++") require.NoError(t, fileutils.MkdirAll(osFS, realDir)) - err := writeAliasSymlink(osFS, realDir, "libxml++") + err := writeAliasSymlink(osFS, realDir, "libxml++", false) require.NoError(t, err) aliasPath := filepath.Join(letterDir, "libxml%2B%2B") @@ -458,7 +459,7 @@ func TestWriteAliasSymlink(t *testing.T) { realDir := filepath.Join(letterDir, "vim") require.NoError(t, fileutils.MkdirAll(osFS, realDir)) - err := writeAliasSymlink(osFS, realDir, "vim") + err := writeAliasSymlink(osFS, realDir, "vim", false) require.NoError(t, err) entries, err := fileutils.ReadDir(osFS, letterDir) @@ -477,7 +478,7 @@ func TestWriteAliasSymlink(t *testing.T) { aliasPath := filepath.Join(letterDir, "gtk%2B") osSymlink(t, osFS, "nonexistent-target", aliasPath) - err := writeAliasSymlink(osFS, realDir, "gtk+") + err := writeAliasSymlink(osFS, realDir, "gtk+", false) require.NoError(t, err) assert.Equal(t, "gtk+", osReadlink(t, osFS, aliasPath), "stale symlink should be replaced") @@ -509,7 +510,7 @@ func TestWriteAliasSymlink(t *testing.T) { osSymlink(t, osFS, "old-target", aliasPath) } - require.NoError(t, writeAliasSymlink(osFS, realDir, "gtk+")) + require.NoError(t, writeAliasSymlink(osFS, realDir, "gtk+", true)) linkInfo := osLstat(t, osFS, aliasPath) assert.NotZero(t, linkInfo.Mode()&os.ModeSymlink) assert.Equal(t, "gtk+", osReadlink(t, osFS, aliasPath)) @@ -527,11 +528,59 @@ func TestWriteAliasSymlink(t *testing.T) { memFS := afero.NewMemMapFs() require.NoError(t, fileutils.MkdirAll(memFS, "/SPECS/l/libxml++")) - err := writeAliasSymlink(memFS, "/SPECS/l/libxml++", "libxml++") + err := writeAliasSymlink(memFS, "/SPECS/l/libxml++", "libxml++", false) assert.NoError(t, err, "must not fail when FS doesn't support symlinks") }) } +type renderAliasTestFS struct { + afero.Fs + linked bool +} + +func (fs *renderAliasTestFS) SymlinkIfPossible(_, _ string) error { + fs.linked = true + + return nil +} + +func TestWriteRenderedAlias_LockModes(t *testing.T) { + for _, lockless := range []bool{false, true} { + for _, directory := range []bool{false, true} { + t.Run(fmt.Sprintf("lockless=%t/directory=%t", lockless, directory), func(t *testing.T) { + env := testutils.NewTestEnv(t) + aliasFS := &renderAliasTestFS{Fs: env.TestFS} + env.TestFS = aliasFS + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("libxml++") + + path := "/output/l/libxml%2B%2B" + if directory { + path = filepath.Join(path, "keep.txt") + } + + require.NoError(t, fileutils.WriteFile(aliasFS, path, []byte("keep"), fileperms.PublicFile)) + + writeRenderedAlias(env.Env, &preparedComponent{ + comp: comp, compOutputDir: "/output/l/libxml++", lockless: lockless, + }) + + assert.Equal(t, lockless, aliasFS.linked) + + exists, err := fileutils.Exists(aliasFS, path) + require.NoError(t, err) + assert.Equal(t, !lockless, exists) + + if !lockless { + content, err := fileutils.ReadFile(aliasFS, path) + require.NoError(t, err) + assert.Equal(t, "keep", string(content)) + } + }) + } + } +} + func TestPruneOrphanRenderedDirs(t *testing.T) { t.Parallel() From 7ad397f02c0f704ed41312ae1e1eb55c90bf0d6a Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 03:33:07 +0000 Subject: [PATCH 55/70] fix(component): copy locked rendered output across filesystems Restore recursive copying with preserved file modes for locked components. Keep rename-based publication and staged Git history for lockless renders. Cover cross-device rename failures, staged-file retention, and permissions with in-memory filesystem tests for both modes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 2 + internal/app/azldev/cmds/component/render.go | 32 ++++++++++-- .../cmds/component/render_internal_test.go | 19 +++---- .../component/render_output_internal_test.go | 52 +++++++++++++++++++ 4 files changed, 92 insertions(+), 13 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 1bcf582f0..4a9a0988f 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -34,6 +34,8 @@ Unlisted components use lock files, lock validation, and fingerprint-derived synthetic history. Both kinds can be processed together. Locked rendering preserves existing non-symlink entries at percent-encoded alias paths and warns about the collision; lockless rendering replaces those entries. +Locked rendered output is copied, so the work and output directories may reside +on different filesystems. Lockless output continues to use moves. A missing or empty file means all components use lock files. Blank lines and surrounding whitespace are ignored; names are not glob patterns. A file read diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index e1d04bfef..b05465895 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -995,8 +995,8 @@ func lstatIfPossible(fileSystem opctx.FS, path string) (os.FileInfo, error) { return fileSystem.Stat(path) //nolint:wrapcheck // Preserve the filesystem error. } -// moveRenderedOutput recreates the output directory and moves all prepared entries -// except '.git' into it. Existing output requires allowOverwrite. +// moveRenderedOutput recreates the output directory and publishes prepared files +// without '.git'. Locked output is copied; lockless output is moved. func moveRenderedOutput(env *azldev.Env, tempDir string, prep *preparedComponent, allowOverwrite bool) error { componentOutputDir := prep.compOutputDir @@ -1037,7 +1037,15 @@ func moveRenderedOutput(env *azldev.Env, tempDir string, prep *preparedComponent return fmt.Errorf("creating output directory %#q:\n%w", componentOutputDir, mkdirErr) } - entries, err := fileutils.ReadDir(env.FS(), tempDir) + if !prep.lockless { + return copyLockedRenderedOutput(env, tempDir, componentOutputDir) + } + + return moveLocklessRenderedOutput(env.FS(), tempDir, componentOutputDir) +} + +func moveLocklessRenderedOutput(fileSystem afero.Fs, tempDir, componentOutputDir string) error { + entries, err := fileutils.ReadDir(fileSystem, tempDir) if err != nil { return fmt.Errorf("reading prepared output directory %#q:\n%w", tempDir, err) } @@ -1050,7 +1058,7 @@ func moveRenderedOutput(env *azldev.Env, tempDir string, prep *preparedComponent sourcePath := filepath.Join(tempDir, entry.Name()) destPath := filepath.Join(componentOutputDir, entry.Name()) - if moveErr := env.FS().Rename(sourcePath, destPath); moveErr != nil { + if moveErr := fileSystem.Rename(sourcePath, destPath); moveErr != nil { return fmt.Errorf("moving rendered entry %#q to %#q:\n%w", sourcePath, destPath, moveErr) } } @@ -1058,6 +1066,22 @@ func moveRenderedOutput(env *azldev.Env, tempDir string, prep *preparedComponent return nil } +func copyLockedRenderedOutput(env *azldev.Env, tempDir, outputDir string) error { + gitDir := filepath.Join(tempDir, ".git") + if err := env.FS().RemoveAll(gitDir); err != nil { + slog.Debug("Failed to remove .git directory", "path", gitDir, "error", err) + } + + options := fileutils.CopyDirOptions{ + CopyFileOptions: fileutils.CopyFileOptions{PreserveFileMode: true}, + } + if err := fileutils.CopyDirRecursive(env, env.FS(), tempDir, outputDir, options); err != nil { + return fmt.Errorf("copying rendered files to %#q:\n%w", outputDir, err) + } + + return nil +} + // findUnexpandedMacro returns the first filename from specFiles that contains // an unexpanded RPM macro (i.e., a literal "%{...}" sequence), or "" if all // macros were resolved. When spectool cannot resolve a macro, it emits the raw diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 0df271f66..705369c08 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -136,11 +136,11 @@ func TestMoveRenderedOutput(t *testing.T) { require.NoError(t, err) assert.True(t, emptyExists) - head, err := fileutils.ReadFile(fs, gitHead) - require.NoError(t, err) - assert.Equal(t, "upstream history", string(head)) - if testCase.wantErr || testCase.dryRun { + head, err := fileutils.ReadFile(fs, gitHead) + require.NoError(t, err) + assert.Equal(t, "upstream history", string(head)) + old, err := fileutils.ReadFile(fs, oldPath) require.NoError(t, err) assert.Equal(t, "old output", string(old)) @@ -148,12 +148,13 @@ func TestMoveRenderedOutput(t *testing.T) { return } - remaining, err := fileutils.ReadDir(fs, workingDir) - require.NoError(t, err) - require.Len(t, remaining, 1) - assert.Equal(t, ".git", remaining[0].Name()) + for name, want := range files { + content, err := fileutils.ReadFile(fs, filepath.Join(workingDir, name)) + require.NoError(t, err) + assert.Equal(t, want, string(content), "locked publication copies instead of moving") + } - for _, path := range []string{oldPath, filepath.Join(outputDir, ".git")} { + for _, path := range []string{oldPath, filepath.Join(outputDir, ".git"), gitHead} { exists, err := fileutils.Exists(fs, path) require.NoError(t, err) assert.False(t, exists) diff --git a/internal/app/azldev/cmds/component/render_output_internal_test.go b/internal/app/azldev/cmds/component/render_output_internal_test.go index cbdbe457a..0ccf0efa1 100644 --- a/internal/app/azldev/cmds/component/render_output_internal_test.go +++ b/internal/app/azldev/cmds/component/render_output_internal_test.go @@ -5,9 +5,11 @@ package component import ( "errors" + "os" "os/exec" "path/filepath" "strconv" + "syscall" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" @@ -190,6 +192,52 @@ func TestRenderNEVRSkipsFirstAndLockedRenders(t *testing.T) { } } +func TestMoveRenderedOutput_LockModes(t *testing.T) { + for _, lockless := range []bool{false, true} { + for _, crossDevice := range []bool{false, true} { + t.Run(strconv.FormatBool(lockless)+"/"+strconv.FormatBool(crossDevice), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, false) + + outputFS := &renderOutputFaultFS{Fs: env.TestFS} + if crossDevice { + outputFS.failure = "cross-device" + } + + env.TestFS = outputFS + require.NoError(t, outputFS.Chmod("/staging/curl/sidecar", fileperms.PublicExecutable)) + + err := moveRenderedOutput(env.Env, "/staging/curl", &preparedComponent{ + compOutputDir: testRenderOutputDir, lockless: lockless, + }, true) + if crossDevice && lockless { + require.ErrorIs(t, err, syscall.EXDEV) + + return + } + + require.NoError(t, err) + + path := filepath.Join(testRenderOutputDir, "sidecar") + content, err := fileutils.ReadFile(outputFS, path) + require.NoError(t, err) + assert.Equal(t, "new sidecar", string(content)) + + info, err := outputFS.Stat(path) + require.NoError(t, err) + assert.Equal(t, fileperms.PublicExecutable, info.Mode().Perm()) + + exists, err := fileutils.Exists(outputFS, "/staging/curl/sidecar") + require.NoError(t, err) + assert.Equal(t, !lockless, exists) + exists, err = fileutils.Exists(outputFS, "/staging/curl/.git/HEAD") + require.NoError(t, err) + assert.Equal(t, lockless, exists) + }) + } + } +} + func stubIncreasingRenderNEVR(t *testing.T, env *testutils.TestEnv) { t.Helper() @@ -485,6 +533,10 @@ type renderOutputFaultFS struct { } func (fs *renderOutputFaultFS) Rename(oldName, newName string) error { + if fs.failure == "cross-device" { + return &os.LinkError{Op: "rename", Old: oldName, New: newName, Err: syscall.EXDEV} + } + if fs.failure == "move" && oldName == "/staging/curl/sidecar" { return errors.New("injected rename failure") } From 4221b3e17bf1c47e445bf380b2bee6a2aba37dd4 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 03:34:51 +0000 Subject: [PATCH 56/70] fix(component): restore complete locked render drift checks Restrict root Git and rebuild-marker exclusions to lockless rendering. Remove staged Git metadata before comparing locked output, including when file filtering is disabled, without changing existing output in check-only. Cover locked and lockless metadata drift, staging cleanup, and nested markers using the render completion path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 3 + internal/app/azldev/cmds/component/render.go | 36 ++++++++---- .../cmds/component/render_internal_test.go | 57 +++++++++++++++++-- 3 files changed, 79 insertions(+), 17 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 4a9a0988f..8467668f9 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -36,6 +36,9 @@ Locked rendering preserves existing non-symlink entries at percent-encoded alias paths and warns about the collision; lockless rendering replaces those entries. Locked rendered output is copied, so the work and output directories may reside on different filesystems. Lockless output continues to use moves. +Only lockless `render --check-only` ignores root `.git` and +`.no-change-rebuild` entries. Locked rendering compares all existing output +entries after removing Git metadata from its staging directory. A missing or empty file means all components use lock files. Blank lines and surrounding whitespace are ignored; names are not glob patterns. A file read diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index b05465895..72a8554cc 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -834,8 +834,8 @@ func finishOneComponent( } // finishComponentRender applies mock results, filters locked components' files, -// diffs the staging tree against the existing output while ignoring the root '.git', -// and (unless checkOnly is set) bumps lockless upstream specs and moves output. +// diffs the staging tree against the existing output with mode-specific exclusions, +// and (unless checkOnly is set) bumps lockless upstream specs and publishes output. // stagingDir is the root staging directory containing the component's subdirectory. // // Returns changed=true when the prepared tree differs from the existing output @@ -866,6 +866,10 @@ func finishComponentRender( "source processing failed for %#q:\n%w", componentName, mockResult.Error) } + if !prep.lockless { + removeRenderedGitMetadata(env, componentDir) + } + // Filter locked components using the mock spectool result. // Skip filtering when: // 1. The component is lockless. @@ -886,7 +890,7 @@ func finishComponentRender( } // Compare before the final bump so '--check-only' does not manufacture release drift. - changed, diffErr := diffRenderedOutput(env.FS(), componentDir, prep.compOutputDir) + changed, diffErr := diffRenderedOutput(env.FS(), componentDir, prep.compOutputDir, prep.lockless) if diffErr != nil { return false, fmt.Errorf("comparing rendered output for %#q:\n%w", componentName, diffErr) } @@ -1067,10 +1071,7 @@ func moveLocklessRenderedOutput(fileSystem afero.Fs, tempDir, componentOutputDir } func copyLockedRenderedOutput(env *azldev.Env, tempDir, outputDir string) error { - gitDir := filepath.Join(tempDir, ".git") - if err := env.FS().RemoveAll(gitDir); err != nil { - slog.Debug("Failed to remove .git directory", "path", gitDir, "error", err) - } + removeRenderedGitMetadata(env, tempDir) options := fileutils.CopyDirOptions{ CopyFileOptions: fileutils.CopyFileOptions{PreserveFileMode: true}, @@ -1082,6 +1083,13 @@ func copyLockedRenderedOutput(env *azldev.Env, tempDir, outputDir string) error return nil } +func removeRenderedGitMetadata(env *azldev.Env, tempDir string) { + gitDir := filepath.Join(tempDir, ".git") + if err := env.FS().RemoveAll(gitDir); err != nil { + slog.Debug("Failed to remove .git directory", "path", gitDir, "error", err) + } +} + // findUnexpandedMacro returns the first filename from specFiles that contains // an unexpanded RPM macro (i.e., a literal "%{...}" sequence), or "" if all // macros were resolved. When spectool cannot resolve a macro, it emits the raw @@ -1342,9 +1350,9 @@ const renderErrorMarkerContent = "Rendering failed. See azldev logs for details. // the existing on-disk output (actualDir) and returns true when they differ. // A missing actualDir always counts as drift. Symlinks are compared by target // (filesystems without symlink support skip that check; matches production -// render behavior). The root '.git' entry is ignored without traversing it. -// The root '.no-change-rebuild' marker is also ignored on both sides. -func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string) (bool, error) { +// render behavior). Only lockless rendering ignores the root '.git' entry +// and '.no-change-rebuild' marker on both sides. +func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string, lockless bool) (bool, error) { actualExists, err := fileutils.DirExists(fileSystem, actualDir) if err != nil { return false, fmt.Errorf("checking actual output dir %#q:\n%w", actualDir, err) @@ -1355,8 +1363,12 @@ func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string) (boo return true, nil } - result, err := dirdiff.DiffDirs(fileSystem, actualDir, expectedDir, - dirdiff.WithIgnoredPaths(".git", noChangeRebuildMarker)) + var options []dirdiff.DiffOption + if lockless { + options = append(options, dirdiff.WithIgnoredPaths(".git", noChangeRebuildMarker)) + } + + result, err := dirdiff.DiffDirs(fileSystem, actualDir, expectedDir, options...) if err != nil { return false, fmt.Errorf("diffing %#q vs %#q:\n%w", actualDir, expectedDir, err) } diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 705369c08..321d88d10 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -16,6 +16,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/spf13/afero" @@ -798,6 +799,52 @@ func TestFinishComponentRender_PreservesGitMetadata(t *testing.T) { } } +func TestFinishComponentRender_CheckOnlyMetadata(t *testing.T) { + for _, lockless := range []bool{false, true} { + for _, metadata := range []string{"", ".git/HEAD", noChangeRebuildMarker, "nested/" + noChangeRebuildMarker} { + t.Run(fmt.Sprintf("lockless=%t/%s", lockless, metadata), func(t *testing.T) { + env := testutils.NewTestEnv(t) + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Render: projectconfig.ComponentRenderConfig{SkipFileFilter: true}, + }) + + for _, root := range []string{"/staging/curl", "/output/curl"} { + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(root, "curl.spec"), + []byte("Name: curl\n"), fileperms.PublicFile)) + } + + const stagedGit = "/staging/curl/.git/HEAD" + require.NoError(t, fileutils.WriteFile(env.TestFS, stagedGit, []byte("upstream"), fileperms.PublicFile)) + + if metadata != "" { + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join("/output/curl", metadata), + []byte("existing metadata"), fileperms.PublicFile)) + } + + prep := &preparedComponent{ + comp: comp, specFilename: "curl.spec", compOutputDir: "/output/curl", lockless: lockless, + } + changed, err := finishComponentRender(env.Env, prep, + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, true, false) + require.NoError(t, err) + assert.Equal(t, metadata != "" && (!lockless || strings.HasPrefix(metadata, "nested/")), changed) + + exists, err := fileutils.Exists(env.TestFS, stagedGit) + require.NoError(t, err) + assert.Equal(t, lockless, exists) + + if metadata != "" { + content, err := fileutils.ReadFile(env.TestFS, filepath.Join("/output/curl", metadata)) + require.NoError(t, err) + assert.Equal(t, "existing metadata", string(content), "check-only must not change existing output") + } + }) + } + } +} + func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { for _, testCase := range []struct { name string @@ -986,7 +1033,7 @@ func TestDiffRenderedOutput(t *testing.T) { require.NoError(t, fileutils.WriteFile(testFS, "/act/curl.spec", []byte("Name: curl"), fileperms.PublicFile)) require.NoError(t, fileutils.WriteFile(testFS, "/act/sources", []byte("hash"), fileperms.PublicFile)) - drifted, err := diffRenderedOutput(testFS, "/exp", "/act") + drifted, err := diffRenderedOutput(testFS, "/exp", "/act", false) require.NoError(t, err) assert.False(t, drifted) }) @@ -997,7 +1044,7 @@ func TestDiffRenderedOutput(t *testing.T) { testFS := afero.NewMemMapFs() require.NoError(t, fileutils.WriteFile(testFS, "/exp/curl.spec", []byte("x"), fileperms.PublicFile)) - drifted, err := diffRenderedOutput(testFS, "/exp", "/missing") + drifted, err := diffRenderedOutput(testFS, "/exp", "/missing", false) require.NoError(t, err) assert.True(t, drifted) }) @@ -1009,7 +1056,7 @@ func TestDiffRenderedOutput(t *testing.T) { require.NoError(t, fileutils.WriteFile(testFS, "/exp/curl.spec", []byte("v2"), fileperms.PublicFile)) require.NoError(t, fileutils.WriteFile(testFS, "/act/curl.spec", []byte("v1"), fileperms.PublicFile)) - drifted, err := diffRenderedOutput(testFS, "/exp", "/act") + drifted, err := diffRenderedOutput(testFS, "/exp", "/act", false) require.NoError(t, err) assert.True(t, drifted) }) @@ -1022,7 +1069,7 @@ func TestDiffRenderedOutput(t *testing.T) { require.NoError(t, fileutils.WriteFile(testFS, "/act/curl.spec", []byte("x"), fileperms.PublicFile)) require.NoError(t, fileutils.WriteFile(testFS, "/act/stale.patch", []byte("y"), fileperms.PublicFile)) - drifted, err := diffRenderedOutput(testFS, "/exp", "/act") + drifted, err := diffRenderedOutput(testFS, "/exp", "/act", false) require.NoError(t, err) assert.True(t, drifted) }) @@ -1060,7 +1107,7 @@ func TestDiffRenderedOutput_RebuildMarker(t *testing.T) { []byte("working timestamp\n"), fileperms.PublicFile)) } - changed, err := diffRenderedOutput(testFS, "/working", "/existing") + changed, err := diffRenderedOutput(testFS, "/working", "/existing", true) require.NoError(t, err) assert.Equal(t, testCase.wantChanged, changed) }) From bfc4bda831559754d72ffd6cbffa861a99fcbad8 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 03:37:44 +0000 Subject: [PATCH 57/70] fix(spec): preserve release formatting only for lockless edits Restore normalized Release tag updates by default in both spec editors. Opt into preserving casing and alignment for lockless preparation, changelog carryover, autorelease transitions, and source overlays. Test both editor modes and mutation APIs, plus source-preparer overlays and generated release bumps in locked and lockless modes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3dc8e9aa-35cd-42a2-9377-a73317ee4a51 --- docs/user/how-to/preview-without-lockfile.md | 2 + .../azldev/cmds/component/render_lockless.go | 4 +- .../component/render_release_carryover.go | 3 +- internal/app/azldev/core/sources/release.go | 2 +- .../core/sources/release_internal_test.go | 34 +++++++++++ .../app/azldev/core/sources/sourceprep.go | 11 +++- internal/rpm/spec/edit_test.go | 56 +++++++++++++------ internal/rpm/spec/editor.go | 19 +++++-- internal/rpm/spec/legacy_edit.go | 2 +- internal/rpm/spec/legacy_spec.go | 7 ++- internal/rpm/spec/structural_edit.go | 2 +- internal/rpm/spec/structural_spec.go | 7 ++- 12 files changed, 114 insertions(+), 35 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 8467668f9..096b69a69 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -39,6 +39,8 @@ on different filesystems. Lockless output continues to use moves. Only lockless `render --check-only` ignores root `.git` and `.no-change-rebuild` entries. Locked rendering compares all existing output entries after removing Git metadata from its staging directory. +Lockless Release edits preserve existing tag casing and alignment; locked +components retain the established `Release: value` normalization. A missing or empty file means all components use lock files. Blank lines and surrounding whitespace are ignored; names are not glob patterns. A file read diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index fa2d136ad..d4ab35cc9 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -379,7 +379,7 @@ func openLocklessSpec( return "", nil, fmt.Errorf("reading upstream spec:\n%w", err) } - opened, err = spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor)) + opened, err = spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor), spec.WithPreservedReleaseFormatting()) if err != nil { return "", nil, fmt.Errorf("parsing upstream spec:\n%w", err) } @@ -469,7 +469,7 @@ func carryLocklessInlineChangelog( return nil, err } - carried, err := spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor)) + carried, err := spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor), spec.WithPreservedReleaseFormatting()) if err != nil { return nil, fmt.Errorf("parsing spec with committed changelog:\n%w", err) } diff --git a/internal/app/azldev/cmds/component/render_release_carryover.go b/internal/app/azldev/cmds/component/render_release_carryover.go index 0d05691fa..71a44d7a5 100644 --- a/internal/app/azldev/cmds/component/render_release_carryover.go +++ b/internal/app/azldev/cmds/component/render_release_carryover.go @@ -30,7 +30,8 @@ func staticRenderHistory( return history, nil } - previous, err := spec.OpenSpec(strings.NewReader(history.spec), spec.WithEditor(editor)) + previous, err := spec.OpenSpec( + strings.NewReader(history.spec), spec.WithEditor(editor), spec.WithPreservedReleaseFormatting()) if err != nil { return history, fmt.Errorf("parsing committed release spec:\n%w", err) } diff --git a/internal/app/azldev/core/sources/release.go b/internal/app/azldev/core/sources/release.go index 6cb225af8..c59faeaaa 100644 --- a/internal/app/azldev/core/sources/release.go +++ b/internal/app/azldev/core/sources/release.go @@ -208,7 +208,7 @@ func (p *sourcePreparerImpl) readAndBumpRelease( Value: newRelease, } - if err := ApplySpecOverlayToFileInPlace(p.fs, overlay, specPath, spec.WithEditor(p.specEditor)); err != nil { + if err := ApplySpecOverlayToFileInPlace(p.fs, overlay, specPath, p.specOptions()...); err != nil { return fmt.Errorf("failed to apply release bump overlay for component %#q:\n%w", component.GetName(), err) } diff --git a/internal/app/azldev/core/sources/release_internal_test.go b/internal/app/azldev/core/sources/release_internal_test.go index 629fc5ef6..8c386f8c6 100644 --- a/internal/app/azldev/core/sources/release_internal_test.go +++ b/internal/app/azldev/core/sources/release_internal_test.go @@ -211,6 +211,40 @@ func TestTryBumpStaticRelease_AutoreleaseSkips(t *testing.T) { require.NoError(t, preparer.tryBumpStaticRelease(comp, filepath.Join(testSourcesDir, "test-pkg"), 3)) } +func TestSourcePreparer_ReleaseFormattingByLockMode(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + for _, lockless := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/lockless=%t", editor, lockless), func(t *testing.T) { + ctx := newBumpspecCtx() + preparer := &sourcePreparerImpl{ + fs: ctx.FS(), dryRunnable: ctx, specEditor: editor, withoutLockfile: lockless, + } + specPath := writeTestSpec(t, ctx.FS(), "curl", "Name: curl\nVersion: 1\nrelease:\t 1%{?dist}\n") + overlay := projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayUpdateSpecTag, Tag: "Release", Value: "2%{?dist}", + } + require.NoError(t, preparer.applyOverlayList([]projectconfig.ComponentOverlay{overlay}, + filepath.Dir(specPath), specPath)) + + prefix := "Release: " + if lockless { + prefix = "release:\t " + } + + content, err := fileutils.ReadFile(ctx.FS(), specPath) + require.NoError(t, err) + assert.Contains(t, string(content), "\n"+prefix+"2%{?dist}\n") + + comp := mockReleaseComponent(gomock.NewController(t), "curl", projectconfig.ReleaseCalculationStatic) + require.NoError(t, preparer.tryBumpStaticRelease(comp, filepath.Dir(specPath), 1)) + content, err = fileutils.ReadFile(ctx.FS(), specPath) + require.NoError(t, err) + assert.Contains(t, string(content), "\n"+prefix+"3%{?dist}\n") + }) + } + } +} + func TestTryBumpStaticRelease_StaticBumps(t *testing.T) { ctrl := gomock.NewController(t) memFS := afero.NewMemMapFs() diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index d5d1563c9..54fbc3958 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -1533,7 +1533,7 @@ func (p *sourcePreparerImpl) applyOverlayList( } if err := ApplyOverlayToSources( - p.dryRunnable, p.fs, overlay, sourcesDirPath, absSpecPath, spec.WithEditor(p.specEditor), + p.dryRunnable, p.fs, overlay, sourcesDirPath, absSpecPath, p.specOptions()..., ); err != nil { return fmt.Errorf("failed to apply %#q overlay:\n%w", overlay.Type, err) } @@ -1542,6 +1542,15 @@ func (p *sourcePreparerImpl) applyOverlayList( return nil } +func (p *sourcePreparerImpl) specOptions() []spec.OpenOption { + options := []spec.OpenOption{spec.WithEditor(p.specEditor)} + if p.withoutLockfile { + options = append(options, spec.WithPreservedReleaseFormatting()) + } + + return options +} + func findSpecInDir( fs opctx.FS, component components.Component, dirPath string, ) (string, error) { diff --git a/internal/rpm/spec/edit_test.go b/internal/rpm/spec/edit_test.go index 6dbca0a13..81caaede4 100644 --- a/internal/rpm/spec/edit_test.go +++ b/internal/rpm/spec/edit_test.go @@ -640,24 +640,46 @@ Name: value func TestUpdateExistingTagUpdatesRepeatedConditionalTags(t *testing.T) { for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { t.Run(string(editor), func(t *testing.T) { - input := []string{ - "Name: example", - "%if 0", - "Release: 1", - "%else", - "Release:\t 2", - "%endif", + for _, formatting := range []struct { + name string + preserve bool + }{{name: "default"}, {name: "preserved", preserve: true}} { + t.Run(formatting.name, func(t *testing.T) { + for _, method := range []string{"update", "set"} { + t.Run(method, func(t *testing.T) { + input := []string{ + "Name: example", "%if 0", "release: 1", "%else", "Release:\t 2", "%endif", + } + + options := []spec.OpenOption{spec.WithEditor(editor)} + if formatting.preserve { + options = append(options, spec.WithPreservedReleaseFormatting()) + } + + specFile, err := spec.OpenSpec(strings.NewReader(strings.Join(input, "\n")+"\n"), options...) + require.NoError(t, err) + + if method == "update" { + err = specFile.UpdateExistingTag("", "Release", "3%{?dist}") + } else { + err = specFile.SetTag("", "Release", "3%{?dist}") + } + + require.NoError(t, err) + + var actual bytes.Buffer + require.NoError(t, specFile.Serialize(&actual)) + + input[2], input[4] = "Release: 3%{?dist}", "Release: 3%{?dist}" + if formatting.preserve { + input[2], input[4] = "release: 3%{?dist}", "Release:\t 3%{?dist}" + } + + assert.Equal(t, strings.Join(input, "\n")+"\n", actual.String()) + }) + } + }) } - specFile, err := spec.OpenSpec(strings.NewReader(strings.Join(input, "\n")+"\n"), spec.WithEditor(editor)) - require.NoError(t, err) - require.NoError(t, specFile.UpdateExistingTag("", "Release", "3%{?dist}")) - - var actual bytes.Buffer - require.NoError(t, specFile.Serialize(&actual)) - - input[2] = "Release: 3%{?dist}" - input[4] = "Release:\t 3%{?dist}" - assert.Equal(t, strings.Join(input, "\n")+"\n", actual.String()) }) } } diff --git a/internal/rpm/spec/editor.go b/internal/rpm/spec/editor.go index b2041bf3b..90db3e589 100644 --- a/internal/rpm/spec/editor.go +++ b/internal/rpm/spec/editor.go @@ -23,7 +23,8 @@ const ( ) type editorOptions struct { - mode EditorMode + mode EditorMode + preserveReleaseFormatting bool } // OpenOption configures [OpenSpec]. @@ -36,6 +37,14 @@ func WithEditor(mode EditorMode) OpenOption { } } +// WithPreservedReleaseFormatting retains existing Release tag casing and spacing +// when updating its value. +func WithPreservedReleaseFormatting() OpenOption { + return func(options *editorOptions) { + options.preserveReleaseFormatting = true + } +} + //nolint:interfacebloat,inamedparam // The facade must cover the established public Spec API. type specEditor interface { Serialize(io.Writer) error @@ -73,8 +82,8 @@ type Spec struct { editor specEditor } -func updatedTagLine(original, tag, value string) string { - if strings.EqualFold(tag, "Release") { +func updatedTagLine(original, tag, value string, preserveReleaseFormatting bool) string { + if preserveReleaseFormatting && strings.EqualFold(tag, "Release") { prefix, rest, found := strings.Cut(original, ":") if found { spacing := rest[:len(rest)-len(strings.TrimLeft(rest, " \t"))] @@ -109,9 +118,9 @@ func OpenSpec(reader io.Reader, options ...OpenOption) (*Spec, error) { switch config.mode { case EditorLegacy, "": - editor, err = openLegacySpec(reader) + editor, err = openLegacySpec(reader, config.preserveReleaseFormatting) case EditorStructural: - editor, err = openStructuralSpec(reader) + editor, err = openStructuralSpec(reader, config.preserveReleaseFormatting) default: return nil, fmt.Errorf("unknown spec editor %#q", config.mode) } diff --git a/internal/rpm/spec/legacy_edit.go b/internal/rpm/spec/legacy_edit.go index c5c6544e7..e9a9207f2 100644 --- a/internal/rpm/spec/legacy_edit.go +++ b/internal/rpm/spec/legacy_edit.go @@ -60,7 +60,7 @@ func (s *legacySpec) UpdateExistingTag(packageName string, tag string, value str return nil } - ctx.ReplaceLine(updatedTagLine(ctx.Target.Line.Text, tag, value)) + ctx.ReplaceLine(updatedTagLine(ctx.Target.Line.Text, tag, value, s.preserveReleaseFormatting)) updated = true diff --git a/internal/rpm/spec/legacy_spec.go b/internal/rpm/spec/legacy_spec.go index 7c3aaabc0..bcaae26a2 100644 --- a/internal/rpm/spec/legacy_spec.go +++ b/internal/rpm/spec/legacy_spec.go @@ -13,7 +13,8 @@ import ( ) type legacySpec struct { - rawLines []string + rawLines []string + preserveReleaseFormatting bool } type parseState struct { @@ -32,9 +33,9 @@ func newParseState() parseState { // OpenSpec reads in the contents of an RPM spec file from the provided reader, returning a [Spec] object. // An error is returned if the reader cannot be fully read (e.g., I/O error or line exceeds buffer size). -func openLegacySpec(reader io.Reader) (*legacySpec, error) { +func openLegacySpec(reader io.Reader, preserveReleaseFormatting bool) (*legacySpec, error) { scanner := bufio.NewScanner(reader) - spec := &legacySpec{} + spec := &legacySpec{preserveReleaseFormatting: preserveReleaseFormatting} // Read each line from the reader, parsing as we go. Store all parsed lines in the spec object. for scanner.Scan() { diff --git a/internal/rpm/spec/structural_edit.go b/internal/rpm/spec/structural_edit.go index 3552b06e1..f64b74c5f 100644 --- a/internal/rpm/spec/structural_edit.go +++ b/internal/rpm/spec/structural_edit.go @@ -52,7 +52,7 @@ func (s *structuralSpec) UpdateExistingTag(packageName string, tag string, value return nil } - line.Replace(updatedTagLine(line.Text, tag, value)) + line.Replace(updatedTagLine(line.Text, tag, value, s.preserveReleaseFormatting)) updated = true diff --git a/internal/rpm/spec/structural_spec.go b/internal/rpm/spec/structural_spec.go index 73cff841d..8731e9ac6 100644 --- a/internal/rpm/spec/structural_spec.go +++ b/internal/rpm/spec/structural_spec.go @@ -52,7 +52,8 @@ var sectionTypesByName = map[string]SectionType{ // Spec encapsulates the contents of an RPM spec file. type structuralSpec struct { - rawLines []string + rawLines []string + preserveReleaseFormatting bool } // Line represents a single line in an RPM spec file. @@ -137,9 +138,9 @@ func (*RawLine) GetType() ParsedLineType { // OpenSpec reads in the contents of an RPM spec file from the provided reader, returning a [Spec] object. // An error is returned if the reader cannot be fully read (e.g., I/O error or line exceeds buffer size). -func openStructuralSpec(reader io.Reader) (*structuralSpec, error) { +func openStructuralSpec(reader io.Reader, preserveReleaseFormatting bool) (*structuralSpec, error) { scanner := bufio.NewScanner(reader) - spec := &structuralSpec{} + spec := &structuralSpec{preserveReleaseFormatting: preserveReleaseFormatting} // Read each line from the reader, parsing as we go. Store all parsed lines in the spec object. for scanner.Scan() { From f7e1efc607c3499901b9b192c59663705d816058 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 05:23:45 +0000 Subject: [PATCH 58/70] fix(component): preserve Release formatting in lockless diff-sources diff-sources builds its source preparer without lockless history options, so Release overlays on lockless components were rewritten as 'Release: ' even though render and build preserve the original casing and spacing. Add a preparer option that requests preserved Release formatting and set it for lockless components in diff-sources. Locked components keep the normalized rewrite. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 16503351-1bf3-4693-96f1-26283e4fc641 --- .../app/azldev/cmds/component/diffsources.go | 17 +++++++++++++-- .../core/sources/release_internal_test.go | 21 +++++++++++++++---- .../app/azldev/core/sources/sourceprep.go | 18 ++++++++++++---- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/internal/app/azldev/cmds/component/diffsources.go b/internal/app/azldev/cmds/component/diffsources.go index bd76370d1..f2e97d3f5 100644 --- a/internal/app/azldev/cmds/component/diffsources.go +++ b/internal/app/azldev/cmds/component/diffsources.go @@ -102,8 +102,7 @@ func DiffComponentSources(env *azldev.Env, options *DiffSourcesOptions) (interfa } preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, - sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), - sources.WithSpecEditor(options.specEditorMode())) + diffSourcesPreparerOptions(env, component.GetName(), distro, options)...) if err != nil { return nil, fmt.Errorf("failed to create source preparer:\n%w", err) } @@ -185,3 +184,17 @@ func shouldColorize(env *azldev.Env) bool { return false } + +func diffSourcesPreparerOptions( + env *azldev.Env, componentName string, distro sourceproviders.ResolvedDistro, options *DiffSourcesOptions, +) []sources.PreparerOption { + preparerOpts := []sources.PreparerOption{ + sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), + sources.WithSpecEditor(options.specEditorMode()), + } + if env.ComponentWithoutLockfile(componentName) { + preparerOpts = append(preparerOpts, sources.WithPreservedReleaseFormatting()) + } + + return preparerOpts +} diff --git a/internal/app/azldev/core/sources/release_internal_test.go b/internal/app/azldev/core/sources/release_internal_test.go index 8c386f8c6..86cdfd912 100644 --- a/internal/app/azldev/core/sources/release_internal_test.go +++ b/internal/app/azldev/core/sources/release_internal_test.go @@ -213,12 +213,25 @@ func TestTryBumpStaticRelease_AutoreleaseSkips(t *testing.T) { func TestSourcePreparer_ReleaseFormattingByLockMode(t *testing.T) { for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { - for _, lockless := range []bool{false, true} { - t.Run(fmt.Sprintf("%s/lockless=%t", editor, lockless), func(t *testing.T) { + for _, mode := range []struct { + name string + lockless bool + preserveExplicit bool + }{ + {name: "locked"}, + {name: "lockless", lockless: true}, + {name: "explicit", preserveExplicit: true}, + } { + t.Run(fmt.Sprintf("%s/%s", editor, mode.name), func(t *testing.T) { ctx := newBumpspecCtx() + preparer := &sourcePreparerImpl{ - fs: ctx.FS(), dryRunnable: ctx, specEditor: editor, withoutLockfile: lockless, + fs: ctx.FS(), dryRunnable: ctx, specEditor: editor, withoutLockfile: mode.lockless, + } + if mode.preserveExplicit { + WithPreservedReleaseFormatting()(preparer) } + specPath := writeTestSpec(t, ctx.FS(), "curl", "Name: curl\nVersion: 1\nrelease:\t 1%{?dist}\n") overlay := projectconfig.ComponentOverlay{ Type: projectconfig.ComponentOverlayUpdateSpecTag, Tag: "Release", Value: "2%{?dist}", @@ -227,7 +240,7 @@ func TestSourcePreparer_ReleaseFormattingByLockMode(t *testing.T) { filepath.Dir(specPath), specPath)) prefix := "Release: " - if lockless { + if mode.lockless || mode.preserveExplicit { prefix = "release:\t " } diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 54fbc3958..9a17818b1 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -135,6 +135,13 @@ func WithSpecEditor(mode spec.EditorMode) PreparerOption { return func(p *sourcePreparerImpl) { p.specEditor = mode } } +// WithPreservedReleaseFormatting returns a [PreparerOption] that keeps existing +// 'Release' tag casing and spacing when overlays edit its value, for lockless +// callers that do not otherwise select [WithoutLockfileHistory]. +func WithPreservedReleaseFormatting() PreparerOption { + return func(p *sourcePreparerImpl) { p.preserveReleaseFormatting = true } +} + // WithSkipLookaside returns a [PreparerOption] that skips all lookaside cache // downloads during source preparation. This includes both explicit source file // downloads ([SourceManager.FetchFiles]) and lookaside extraction during @@ -215,9 +222,12 @@ type sourcePreparerImpl struct { sourceManager sourceproviders.SourceManager prepareFetchedSources func(context.Context, components.Component, string) error specEditor spec.EditorMode - fs opctx.FS - eventListener opctx.EventListener - dryRunnable opctx.DryRunnable + // preserveReleaseFormatting keeps 'Release' tag formatting during overlay + // edits. Set via [WithPreservedReleaseFormatting]. + preserveReleaseFormatting bool + fs opctx.FS + eventListener opctx.EventListener + dryRunnable opctx.DryRunnable // withGitRepo, when true, enables dist-git creation by preserving the // upstream .git directory and generating synthetic commit history. @@ -1544,7 +1554,7 @@ func (p *sourcePreparerImpl) applyOverlayList( func (p *sourcePreparerImpl) specOptions() []spec.OpenOption { options := []spec.OpenOption{spec.WithEditor(p.specEditor)} - if p.withoutLockfile { + if p.withoutLockfile || p.preserveReleaseFormatting { options = append(options, spec.WithPreservedReleaseFormatting()) } From 9722e3aa59c2cd70df46b272e7c4e2a1bd95d125 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 06:13:45 +0000 Subject: [PATCH 59/70] feat(spec): add per-tag Release update and lookup helpers Add UpdateExistingTagFunc, which computes each matching tag's new value from its current value, and GetTags, which returns every matching tag value in spec order. Both are implemented for the legacy and structural editors and exposed on the Spec facade. UpdateExistingTag is now a wrapper that returns the same value for every tag, so its output and errors are unchanged for existing callers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 16503351-1bf3-4693-96f1-26283e4fc641 --- internal/rpm/spec/edit_test.go | 46 ++++++++++++++++++++++ internal/rpm/spec/editor.go | 19 +++++++++ internal/rpm/spec/legacy_edit.go | 49 +++++++++++++++++++++++- internal/rpm/spec/structural_edit.go | 39 ++++++++++++++++++- internal/rpm/spec/structural_tree_api.go | 20 ++++++---- 5 files changed, 162 insertions(+), 11 deletions(-) diff --git a/internal/rpm/spec/edit_test.go b/internal/rpm/spec/edit_test.go index 81caaede4..32644691f 100644 --- a/internal/rpm/spec/edit_test.go +++ b/internal/rpm/spec/edit_test.go @@ -684,6 +684,52 @@ func TestUpdateExistingTagUpdatesRepeatedConditionalTags(t *testing.T) { } } +func TestUpdateExistingTagFuncUpdatesEachTag(t *testing.T) { + errUpdate := errors.New("update failed") + + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + input := []string{ + "Name: example", "%if 0", "Release: 1", "%else", "Release:\t 2.%{snap}", "%endif", + } + + specFile, err := spec.OpenSpec(strings.NewReader(strings.Join(input, "\n")+"\n"), + spec.WithEditor(editor), spec.WithPreservedReleaseFormatting()) + require.NoError(t, err) + + releases, err := specFile.GetTags("", "Release") + require.NoError(t, err) + assert.Equal(t, []string{"1", "2.%{snap}"}, releases) + + var seen []string + + require.NoError(t, specFile.UpdateExistingTagFunc("", "Release", func(current string) (string, error) { + seen = append(seen, current) + + return "1" + current, nil + })) + assert.Equal(t, []string{"1", "2.%{snap}"}, seen) + + var actual bytes.Buffer + require.NoError(t, specFile.Serialize(&actual)) + + input[2], input[4] = "Release: 11", "Release:\t 12.%{snap}" + assert.Equal(t, strings.Join(input, "\n")+"\n", actual.String()) + + err = specFile.UpdateExistingTagFunc("", "Release", func(string) (string, error) { + return "", errUpdate + }) + require.ErrorIs(t, err, errUpdate) + + _, err = specFile.GetTags("", "Epoch") + require.ErrorIs(t, err, spec.ErrNoSuchTag) + require.ErrorIs(t, specFile.UpdateExistingTagFunc("", "Epoch", func(string) (string, error) { + return "1", nil + }), spec.ErrNoSuchTag) + }) + } +} + func TestRemoveTag(t *testing.T) { tests := []struct { name string diff --git a/internal/rpm/spec/editor.go b/internal/rpm/spec/editor.go index 90db3e589..73381ad08 100644 --- a/internal/rpm/spec/editor.go +++ b/internal/rpm/spec/editor.go @@ -57,6 +57,7 @@ type specEditor interface { VisitTagsPackage(string, func(*TagLine, *Context) error) error SetTag(string, string, string) error UpdateExistingTag(string, string, string) error + UpdateExistingTagFunc(string, string, func(string) (string, error)) error RemoveTag(string, string, string) error RemoveTagsMatching(string, func(string, string) bool) (int, error) AddTag(string, string, string) error @@ -75,6 +76,7 @@ type specEditor interface { RemoveSubpackage(string) error GetTag(string, string) (string, error) GetLastTag(string, string) (string, error) + GetTags(string, string) ([]string, error) } // Spec is the public facade for a configuration-selected RPM spec editor. @@ -190,6 +192,15 @@ func (s *Spec) UpdateExistingTag(pkg, tag, value string) error { return s.activeEditor().UpdateExistingTag(pkg, tag, value) } +// UpdateExistingTagFunc updates every matching tag in the specified package to the +// value returned by update for that tag's current value. It stops at the first +// error returned by update. Returns [ErrNoSuchTag] if no matching tag exists. +// +//nolint:wrapcheck // Preserve errors returned by the selected editor. +func (s *Spec) UpdateExistingTagFunc(pkg, tag string, update func(current string) (string, error)) error { + return s.activeEditor().UpdateExistingTagFunc(pkg, tag, update) +} + // RemoveTag removes matching tags from the specified package. // //nolint:wrapcheck // Preserve errors returned by the selected editor. @@ -301,6 +312,14 @@ func (s *Spec) GetTag(pkg, tag string) (string, error) { return s.activeEditor().GetTag(pkg, tag) } +// GetTags returns the values of every matching tag in the specified package, in +// spec order. Returns [ErrNoSuchTag] if no matching tag exists. +// +//nolint:wrapcheck // Preserve errors returned by the selected editor. +func (s *Spec) GetTags(pkg, tag string) ([]string, error) { + return s.activeEditor().GetTags(pkg, tag) +} + // GetLastTag returns the final matching tag in the specified package. // //nolint:wrapcheck // Preserve errors returned by the selected editor. diff --git a/internal/rpm/spec/legacy_edit.go b/internal/rpm/spec/legacy_edit.go index e9a9207f2..be45931a4 100644 --- a/internal/rpm/spec/legacy_edit.go +++ b/internal/rpm/spec/legacy_edit.go @@ -48,18 +48,37 @@ func (s *legacySpec) SetTag(packageName string, tag string, value string) (err e // UpdateExistingTag looks for the first instance of the named tag in the given package; if it // finds such a tag instance, it replaces its value with the provided value. If no such tag // exists, it returns an error. -func (s *legacySpec) UpdateExistingTag(packageName string, tag string, value string) (err error) { +func (s *legacySpec) UpdateExistingTag(packageName string, tag string, value string) error { slog.Debug("Updating tag in spec", "package", packageName, "tag", tag, "newValue", value) + return s.UpdateExistingTagFunc(packageName, tag, func(string) (string, error) { return value, nil }) +} + +// UpdateExistingTagFunc replaces the value of every instance of the named tag in the given +// package with the value returned by update for that instance's current value. If no such +// tag exists, it returns an error. +func (s *legacySpec) UpdateExistingTagFunc( + packageName string, tag string, update func(current string) (string, error), +) (err error) { tagToCompareAgainst := strings.ToLower(tag) - var updated bool + var ( + updated bool + updateErr error + ) err = s.VisitTagsPackage(packageName, func(tagLine *TagLine, ctx *Context) error { if strings.ToLower(tagLine.Tag) != tagToCompareAgainst { return nil } + var value string + + value, updateErr = update(tagLine.Value) + if updateErr != nil { + return updateErr + } + ctx.ReplaceLine(updatedTagLine(ctx.Target.Line.Text, tag, value, s.preserveReleaseFormatting)) updated = true @@ -67,6 +86,10 @@ func (s *legacySpec) UpdateExistingTag(packageName string, tag string, value str return nil }) + if updateErr != nil { + return updateErr + } + if !updated { return fmt.Errorf("tag %#q not found in spec:\n%w", tag, ErrNoSuchTag) } @@ -1166,6 +1189,28 @@ func (s *legacySpec) GetTag(packageName, tag string) (string, error) { return value, nil } +// GetTags returns the values of all matching tags in a package, in spec order. +func (s *legacySpec) GetTags(packageName, tag string) ([]string, error) { + var values []string + + err := s.VisitTagsPackage(packageName, func(tagLine *TagLine, _ *Context) error { + if strings.EqualFold(tagLine.Tag, tag) { + values = append(values, tagLine.Value) + } + + return nil + }) + if err != nil { + return nil, err + } + + if len(values) == 0 { + return nil, fmt.Errorf("tag %#q not found in package %#q:\n%w", tag, packageName, ErrNoSuchTag) + } + + return values, nil +} + // GetLastTag returns the last matching tag in a package. func (s *legacySpec) GetLastTag(packageName, tag string) (string, error) { var value string diff --git a/internal/rpm/spec/structural_edit.go b/internal/rpm/spec/structural_edit.go index f64b74c5f..fb71dffea 100644 --- a/internal/rpm/spec/structural_edit.go +++ b/internal/rpm/spec/structural_edit.go @@ -34,9 +34,18 @@ func (s *structuralSpec) SetTag(packageName string, tag string, value string) (e // UpdateExistingTag replaces every instance of the named tag in the given // package with the provided value. If no such tag exists, it returns an error. -func (s *structuralSpec) UpdateExistingTag(packageName string, tag string, value string) (err error) { +func (s *structuralSpec) UpdateExistingTag(packageName string, tag string, value string) error { slog.Debug("Updating tag in spec", "package", packageName, "tag", tag, "newValue", value) + return s.UpdateExistingTagFunc(packageName, tag, func(string) (string, error) { return value, nil }) +} + +// UpdateExistingTagFunc replaces the value of every instance of the named tag in the given +// package with the value returned by update for that instance's current value. If no such +// tag exists, it returns an error. +func (s *structuralSpec) UpdateExistingTagFunc( + packageName string, tag string, update func(current string) (string, error), +) (err error) { tagToCompareAgainst := strings.ToLower(tag) var updated bool @@ -47,11 +56,16 @@ func (s *structuralSpec) UpdateExistingTag(packageName string, tag string, value return nil } - parsedTag, _, isTag := parseTagLine(line.Text) + parsedTag, current, isTag := parseTagLine(line.Text) if !isTag || strings.ToLower(parsedTag) != tagToCompareAgainst { return nil } + value, updateErr := update(current) + if updateErr != nil { + return updateErr + } + line.Replace(updatedTagLine(line.Text, tag, value, s.preserveReleaseFormatting)) updated = true @@ -192,6 +206,27 @@ func (s *structuralSpec) GetTag(packageName string, tag string) (string, error) return foundValue, nil } +// GetTags returns the values of all instances of the named tag in the given +// package, in spec order. Returns [ErrNoSuchTag] if the tag does not exist. +func (s *structuralSpec) GetTags(packageName string, tag string) ([]string, error) { + var values []string + + err := s.inspectTree(func(tree *specTree) error { + values = tree.GetTags(packageName, tag) + + return nil + }) + if err != nil { + return nil, err + } + + if len(values) == 0 { + return nil, fmt.Errorf("tag %#q not found in package %#q:\n%w", tag, packageName, ErrNoSuchTag) + } + + return values, nil +} + // GetLastTag returns the value of the final lexical instance of the named tag // in the given package. Returns [ErrNoSuchTag] if the tag does not exist. func (s *structuralSpec) GetLastTag(packageName string, tag string) (string, error) { diff --git a/internal/rpm/spec/structural_tree_api.go b/internal/rpm/spec/structural_tree_api.go index 5d775421b..82ca03b1b 100644 --- a/internal/rpm/spec/structural_tree_api.go +++ b/internal/rpm/spec/structural_tree_api.go @@ -97,10 +97,17 @@ func (t *specTree) GetTag(pkg, name string) (string, bool) { // GetLastTag returns the last lexical tag matching name in the requested package. func (t *specTree) GetLastTag(pkg, name string) (string, bool) { - var ( - value string - found bool - ) + values := t.GetTags(pkg, name) + if len(values) == 0 { + return "", false + } + + return values[len(values)-1], true +} + +// GetTags returns the values of all instances of the named tag in pkg, in lexical order. +func (t *specTree) GetTags(pkg, name string) []string { + var values []string _ = t.VisitAllLines(func(secName, secPkg string, line *lineHandle) error { if secPkg != pkg || !isTagBearingSection(secName) { @@ -109,14 +116,13 @@ func (t *specTree) GetLastTag(pkg, name string) (string, bool) { tag, tagValue, isTag := parseTagLine(line.Text) if isTag && strings.EqualFold(tag, name) { - value = tagValue - found = true + values = append(values, tagValue) } return nil }) - return value, found + return values } // HasSection reports whether a section with name is present for any package. From ef924d01736bb9d735e1293b2be341366fc96c3d Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 06:13:45 +0000 Subject: [PATCH 60/70] fix(component): render each lockless Release tag separately Lockless rendering read only the last Release tag and wrote its updated value to every Release tag. Specs with conditional Release tags, such as yara's '%if %{with release}' / '%else' pair, had every branch replaced by the last one, changing the default build's NEVR. Compute the autorelease base once and apply it to each Release tag, keeping each tag's own options. Static carryover now copies committed Release values by position and fails when the upstream and committed tag counts differ. Specs that mix '%autorelease' and static Release tags, and committed specs with differing '%autorelease' tags converted to a static release, now fail with a specific error. Locked components do not use this code path. Lockless specs with a single Release tag render exactly as before. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 16503351-1bf3-4693-96f1-26283e4fc641 --- docs/user/how-to/preview-without-lockfile.md | 9 ++ .../azldev/cmds/component/render_lockless.go | 142 +++++++++++++----- .../render_lockless_internal_test.go | 92 +++++++++++- .../component/render_release_carryover.go | 79 ++++++++-- 4 files changed, 275 insertions(+), 47 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 096b69a69..ac99de9cc 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -288,6 +288,15 @@ working static release and `baserelease` definition are left as supplied by upstream and overlays. Autorelease follows the case-specific rules below. There is no later restoration of the original upstream release. +A spec may define several `Release:` tags, such as one per `%if`/`%else` +branch. Each tag is updated separately and keeps its own value and options. +All `Release:` tags in a spec must either use `%autorelease` or be static; +mixing the two fails rendering. Static carryover copies each committed +`Release:` value to the upstream tag at the same position, and fails if the +upstream and committed specs have different numbers of `Release:` tags. When +upstream switches from `%autorelease` to a static release, every committed +`%autorelease` tag must have the same value. + When release carryover runs for an existing non-`%autorelease` upstream spec, it checks the committed `Release:` value. If it references the `baserelease` macro, the renderer copies the committed `%global baserelease` or `%define baserelease` diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index d4ab35cc9..2e6bca9e6 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -396,9 +396,9 @@ func prepareLocklessSpec( return err } - original, err := opened.GetLastTag("", "Release") + original, err := upstreamRenderRelease(opened) if err != nil { - return fmt.Errorf("reading upstream Release:\n%w", err) + return err } version, err := opened.GetLastTag("", "Version") @@ -426,12 +426,12 @@ func prepareLocklessSpec( } if !versionChanged || sources.ReleaseUsesAutorelease(original) { - release, err := locklessRenderRelease(env, autospec, dir, filepath.Base(path), original, history, versionChanged) + update, err := locklessRenderRelease(env, autospec, dir, filepath.Base(path), original, history, versionChanged) if err != nil { return err } - if err := opened.SetTag("", "Release", release); err != nil { + if err := opened.UpdateExistingTagFunc("", "Release", update); err != nil { return fmt.Errorf("updating upstream Release:\n%w", err) } @@ -452,6 +452,42 @@ func prepareLocklessSpec( return nil } +// upstreamRenderRelease returns the final upstream 'Release' value after +// verifying that every 'Release' tag agrees on whether it uses '%autorelease'. +func upstreamRenderRelease(opened *spec.Spec) (string, error) { + releases, err := opened.GetTags("", "Release") + if err != nil { + return "", fmt.Errorf("reading upstream Release:\n%w", err) + } + + if err := checkReleaseTagsConsistent(releases); err != nil { + return "", fmt.Errorf("reading upstream Release:\n%w", err) + } + + return releases[len(releases)-1], nil +} + +// ErrMixedReleaseTags is returned when some, but not all, 'Release' tags in a +// spec use '%autorelease'. +var ErrMixedReleaseTags = errors.New("'Release' tags mix '%autorelease' and static values") + +func checkReleaseTagsConsistent(releases []string) error { + autoreleases := 0 + + for _, release := range releases { + if sources.ReleaseUsesAutorelease(release) { + autoreleases++ + } + } + + if autoreleases != 0 && autoreleases != len(releases) { + return fmt.Errorf("%d of %d 'Release' tags use '%%autorelease':\n%w", + autoreleases, len(releases), ErrMixedReleaseTags) + } + + return nil +} + func carryLocklessInlineChangelog( opened *spec.Spec, previous, release string, editor spec.EditorMode, ) (*spec.Spec, error) { @@ -544,24 +580,52 @@ func resolveRenderUpstreamCommit(env *azldev.Env, dir, commit string) error { return nil } +// releaseUpdate computes the new value of one 'Release' tag from its current value. +type releaseUpdate func(release string) (string, error) + +func keepRelease(release string) (string, error) { + return release, nil +} + +// autoreleaseBaseUpdate returns a [releaseUpdate] that applies the given +// autorelease base to each '%autorelease' 'Release' tag, preserving any other +// arguments on that tag. The base must already be validated. +func autoreleaseBaseUpdate(output string, omitDefault bool) releaseUpdate { + return func(release string) (string, error) { + if !sources.ReleaseUsesAutorelease(release) { + return release, nil + } + + return withAutoreleaseBase(release, output, omitDefault) + } +} + +func stripAutoreleaseBase(release string) (string, error) { + return autoreleaseBaseArg.ReplaceAllString(release, ""), nil +} + func locklessRenderRelease( env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, versionChanged bool, -) (string, error) { +) (releaseUpdate, error) { if !sources.ReleaseUsesAutorelease(release) || env.DryRun() { - return release, nil + return keepRelease, nil } if history.spec != "" && (!versionChanged || renderedAutoreleaseMarker.MatchString(history.spec)) { - return carryRenderedAutorelease(env, autospec, dir, filename, release, history) + return carryRenderedAutorelease(env, autospec, dir, filename, history) } calculated, err := autospec.CalculateBaseRelease(env, dir, filename) if err != nil { - return "", fmt.Errorf("calculating upstream release base:\n%w", err) + return nil, fmt.Errorf("calculating upstream release base:\n%w", err) + } + + if _, _, err := parseAutoreleaseBase(calculated); err != nil { + return nil, err } - return withAutoreleaseBase(release, calculated, true) + return autoreleaseBaseUpdate(calculated, true), nil } var ( @@ -571,50 +635,50 @@ var ( ) func carryRenderedAutorelease( - env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, -) (string, error) { + env *azldev.Env, autospec renderAutorelease, dir, filename string, history lastRender, +) (releaseUpdate, error) { if !renderedAutoreleaseMarker.MatchString(history.spec) { - return carryUnprocessedAutorelease(release, history) + return carryUnprocessedAutorelease(history) } expanded, err := expandCommittedRelease(env, autospec, dir, filename, history) if err != nil { - return "", fmt.Errorf("expanding committed Release:\n%w", err) + return nil, fmt.Errorf("expanding committed Release:\n%w", err) } - release, err = withAutoreleaseBase(release, renderedReleaseNumber.FindString(strings.TrimSpace(expanded)), true) - if err != nil { - return "", fmt.Errorf("using committed Release %#q as autorelease base:\n%w", expanded, err) + base := renderedReleaseNumber.FindString(strings.TrimSpace(expanded)) + if _, _, err := parseAutoreleaseBase(base); err != nil { + return nil, fmt.Errorf("using committed Release %#q as autorelease base:\n%w", expanded, err) } - return release, nil + return autoreleaseBaseUpdate(base, true), nil } -func carryUnprocessedAutorelease(release string, history lastRender) (string, error) { +func carryUnprocessedAutorelease(history lastRender) (releaseUpdate, error) { previous, err := spec.OpenSpec(strings.NewReader(history.spec)) if err != nil { - return "", fmt.Errorf("parsing committed autorelease spec:\n%w", err) + return nil, fmt.Errorf("parsing committed autorelease spec:\n%w", err) } previousRelease, err := previous.GetLastTag("", "Release") if err != nil { - return "", fmt.Errorf("reading committed Release:\n%w", err) + return nil, fmt.Errorf("reading committed Release:\n%w", err) } if !sources.ReleaseUsesAutorelease(previousRelease) { - return release, nil + return keepRelease, nil } - if matches := renderedAutoreleaseBase.FindStringSubmatch(previousRelease); matches != nil { - release, err = withAutoreleaseBase(release, matches[1], false) - if err != nil { - return "", fmt.Errorf("using committed autorelease base:\n%w", err) - } - } else { - release = autoreleaseBaseArg.ReplaceAllString(release, "") + matches := renderedAutoreleaseBase.FindStringSubmatch(previousRelease) + if matches == nil { + return stripAutoreleaseBase, nil } - return release, nil + if _, _, err := parseAutoreleaseBase(matches[1]); err != nil { + return nil, fmt.Errorf("using committed autorelease base:\n%w", err) + } + + return autoreleaseBaseUpdate(matches[1], false), nil } func prepareLocklessChangelog( @@ -787,17 +851,27 @@ func locklessVersionChanged(version string, history lastRender) (bool, error) { var autoreleaseBaseArg = regexp.MustCompile(`(^|\s)-b(?:\s+|=)?[0-9]+`) -func withAutoreleaseBase(release, output string, omitDefault bool) (string, error) { +// parseAutoreleaseBase returns the positive release number at the end of output. +func parseAutoreleaseBase(output string) (number string, value int, err error) { fields := strings.Fields(output) if len(fields) == 0 { - return "", errors.New("empty autorelease base") + return "", 0, errors.New("empty autorelease base") } - number := fields[len(fields)-1] + number = fields[len(fields)-1] - value, err := strconv.Atoi(number) + value, err = strconv.Atoi(number) if err != nil || value < 1 { - return "", fmt.Errorf("invalid autorelease base %#q", output) + return "", 0, fmt.Errorf("invalid autorelease base %#q", output) + } + + return number, value, nil +} + +func withAutoreleaseBase(release, output string, omitDefault bool) (string, error) { + number, value, err := parseAutoreleaseBase(output) + if err != nil { + return "", err } if omitDefault && value == 1 { diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index d21b56b88..bf756eadf 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -1060,7 +1060,9 @@ func TestLocklessRenderRelease_StaticHistoryWithoutAutoreleaseMarker(t *testing. env := testutils.NewTestEnv(t) stub := &renderAutospecStub{} history := lastRender{spec: testCase.header + renderTestSpec("1", "9", "old\n")} - got, err := locklessRenderRelease(env.Env, stub, "/working", "curl.spec", testCase.release, history, false) + update, err := locklessRenderRelease(env.Env, stub, "/working", "curl.spec", testCase.release, history, false) + require.NoError(t, err) + got, err := update(testCase.release) require.NoError(t, err) assert.Equal(t, testCase.release, got) assert.Empty(t, stub.calls) @@ -2080,3 +2082,91 @@ func TestCommitLocklessRender_PathOnlyAndNoEmptyCommits(t *testing.T) { }) } } + +func conditionalReleaseTestSpec(withRelease, withoutRelease string) string { + return fmt.Sprintf("Name: curl\nVersion: 1\n%%if %%{with release}\nRelease: %s\n%%else\n"+ + "Release: %s\n%%endif\nSummary: Test\nLicense: MIT\n\n%%description\nTest\n\n%%changelog\n%%autochangelog\n", + withRelease, withoutRelease) +} + +func TestPrepareLocklessSpec_ConditionalReleaseTags(t *testing.T) { + const snapshot = "%autorelease -s %{gitdate}git%{shortcommit}" + + unprocessed := conditionalReleaseTestSpec("%autorelease -b 5", snapshot+" -b 5") + unprocessedRepo, unprocessedCommits := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + "curl.spec": []byte(unprocessed), "changelog": []byte("committed changelog"), + }}}) + unprocessedTree, err := resolveTree(unprocessedRepo, unprocessedCommits[0]) + require.NoError(t, err) + + for _, testCase := range []struct { + name, upstreamFirst, upstreamSecond string + history lastRender + want []string + wantErr error + }{ + { + name: "initial autorelease", upstreamFirst: "%autorelease", upstreamSecond: snapshot, + want: []string{"%autorelease -b 42", snapshot + " -b 42"}, + }, + { + name: "unprocessed autorelease history", upstreamFirst: "%autorelease", upstreamSecond: snapshot, + history: lastRender{spec: unprocessed, tree: unprocessedTree}, + want: []string{"%autorelease -b 5", snapshot + " -b 5"}, + }, + { + name: "rendered autorelease history", upstreamFirst: "%autorelease", upstreamSecond: snapshot, + history: testCommittedAutoreleaseHistory(t), + want: []string{"%autorelease -b 9", snapshot + " -b 9"}, + }, + { + name: "static release history", upstreamFirst: "1%{?dist}", upstreamSecond: "1.%{snap}%{?dist}", + history: lastRender{spec: conditionalReleaseTestSpec("5%{?dist}", "5.%{snap}%{?dist}")}, + want: []string{"5%{?dist}", "5.%{snap}%{?dist}"}, + }, + { + name: "static release count mismatch", upstreamFirst: "1%{?dist}", upstreamSecond: "1.%{snap}%{?dist}", + history: lastRender{spec: renderTestSpec("1", "5%{?dist}", "")}, + wantErr: ErrReleaseTagCountMismatch, + }, + { + name: "mixed upstream release", upstreamFirst: "%autorelease", upstreamSecond: "1%{?dist}", + wantErr: ErrMixedReleaseTags, + }, + { + name: "differing committed autorelease", upstreamFirst: "1%{?dist}", upstreamSecond: "1.%{snap}%{?dist}", + history: lastRender{spec: conditionalReleaseTestSpec("%autorelease", snapshot)}, + wantErr: ErrDifferingAutoreleaseTags, + }, + } { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(testCase.name+"/"+string(editor), func(t *testing.T) { + env := testutils.NewTestEnv(t) + original := conditionalReleaseTestSpec(testCase.upstreamFirst, testCase.upstreamSecond) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(original), fileperms.PublicFile)) + + stub := &renderAutospecStub{expand: func(_, _ string) (string, error) { return "9.fc43", nil }} + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", editor, testCase.history) + + content, readErr := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, readErr) + + if testCase.wantErr != nil { + require.ErrorIs(t, err, testCase.wantErr) + assert.Equal(t, original, string(content)) + + return + } + + require.NoError(t, err) + + prepared, err := spec.OpenSpec(strings.NewReader(string(content)), spec.WithEditor(editor)) + require.NoError(t, err) + releases, err := prepared.GetTags("", "Release") + require.NoError(t, err) + assert.Equal(t, testCase.want, releases) + }) + } + } +} diff --git a/internal/app/azldev/cmds/component/render_release_carryover.go b/internal/app/azldev/cmds/component/render_release_carryover.go index 71a44d7a5..a5b59b764 100644 --- a/internal/app/azldev/cmds/component/render_release_carryover.go +++ b/internal/app/azldev/cmds/component/render_release_carryover.go @@ -36,12 +36,12 @@ func staticRenderHistory( return history, fmt.Errorf("parsing committed release spec:\n%w", err) } - previousRelease, err := previous.GetLastTag("", "Release") + previousReleases, err := committedReleases(previous) if err != nil { - return history, fmt.Errorf("reading committed Release:\n%w", err) + return history, err } - if !sources.ReleaseUsesAutorelease(previousRelease) { + if !sources.ReleaseUsesAutorelease(previousReleases[0]) { return history, nil } @@ -71,11 +71,11 @@ func staticRenderHistory( } release = fields[len(fields)-1] - if !autoreleaseNoDist.MatchString(previousRelease) { + if !autoreleaseNoDist.MatchString(previousReleases[0]) { release += "%{?dist}" } - if err := previous.SetTag("", "Release", release); err != nil { + if err := previous.UpdateExistingTag("", "Release", release); err != nil { return history, fmt.Errorf("carrying calculated autorelease:\n%w", err) } @@ -123,17 +123,13 @@ func carryStaticRenderRelease( return fmt.Errorf("parsing committed release spec:\n%w", err) } - oldRelease, err := previous.GetLastTag("", "Release") + oldReleases, err := previous.GetTags("", "Release") if err != nil { return fmt.Errorf("reading committed Release:\n%w", err) } - if !baseReleaseMacro.MatchString(oldRelease) { - if err := working.SetTag("", "Release", oldRelease); err != nil { - return fmt.Errorf("carrying committed Release:\n%w", err) - } - - return nil + if !baseReleaseMacro.MatchString(oldReleases[len(oldReleases)-1]) { + return carryCommittedReleases(working, oldReleases) } _, definition, err := baseReleaseDefinition(previous) @@ -151,6 +147,65 @@ func carryStaticRenderRelease( return nil } +// ErrDifferingAutoreleaseTags is returned when a committed spec has several +// '%autorelease' 'Release' tags with different values. +var ErrDifferingAutoreleaseTags = errors.New("committed '%autorelease' 'Release' tags differ") + +// ErrReleaseTagCountMismatch is returned when the upstream and committed specs +// have different numbers of 'Release' tags. +var ErrReleaseTagCountMismatch = errors.New("upstream and committed 'Release' tag counts differ") + +// committedReleases returns the committed 'Release' values, requiring them all +// to be static or all to be the same '%autorelease' value. +func committedReleases(previous *spec.Spec) ([]string, error) { + releases, err := previous.GetTags("", "Release") + if err != nil { + return nil, fmt.Errorf("reading committed Release:\n%w", err) + } + + if err := checkReleaseTagsConsistent(releases); err != nil { + return nil, fmt.Errorf("reading committed Release:\n%w", err) + } + + if sources.ReleaseUsesAutorelease(releases[0]) { + for _, release := range releases[1:] { + if release != releases[0] { + return nil, fmt.Errorf("calculating committed autorelease:\n%w", ErrDifferingAutoreleaseTags) + } + } + } + + return releases, nil +} + +// carryCommittedReleases copies each committed 'Release' value onto the +// upstream 'Release' tag at the same position. +func carryCommittedReleases(working *spec.Spec, oldReleases []string) error { + releases, err := working.GetTags("", "Release") + if err != nil { + return fmt.Errorf("reading upstream Release:\n%w", err) + } + + if len(releases) != len(oldReleases) { + return fmt.Errorf("carrying committed Release: upstream spec has %d 'Release' tags, committed spec has %d:\n%w", + len(releases), len(oldReleases), ErrReleaseTagCountMismatch) + } + + index := 0 + + err = working.UpdateExistingTagFunc("", "Release", func(string) (string, error) { + release := oldReleases[index] + index++ + + return release, nil + }) + if err != nil { + return fmt.Errorf("carrying committed Release:\n%w", err) + } + + return nil +} + func baseReleaseDefinition(opened *spec.Spec) (line int, definition string, err error) { var matches int From f52631f7084c16dc31d9ca2583ca8f413094b278 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 03:37:50 +0000 Subject: [PATCH 61/70] fix(component): parse historical config permissively Historical project snapshots are written against older config schemas. Strictly decoding them fails once azldev drops a field, for example the removed [test-suites] table in older images.toml files. That broke the lockless render upstream history lookup for every component last rendered before the removal, so render commits fell back to a generic "Update to version " message instead of the upstream log. Always ignore unknown fields when loading historical configuration. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8b620ba1-9249-46b7-bfb4-78616d6f1ec1 --- .../cmds/component/changed_upstreamcommit.go | 8 +++-- .../changed_upstreamcommit_internal_test.go | 32 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit.go b/internal/app/azldev/cmds/component/changed_upstreamcommit.go index 68f338c3b..92a3b31b4 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit.go @@ -182,6 +182,10 @@ func loadHistoricalEnv( projectDir := filepath.Join(snapshotRepoRoot, projectRelDir) + // Historical snapshots may use config fields that this azldev no longer + // supports, so always parse them permissively. + const permissive = true + loadedProjectDir, config, err := projectconfig.LoadProjectConfig( snapshotFS, env.OSEnv(), @@ -189,7 +193,7 @@ func loadHistoricalEnv( false, snapshotTempDir, nil, - env.PermissiveConfigParsing(), + permissive, true, /*withoutLockfile*/ ) if err != nil { @@ -204,7 +208,7 @@ func loadHistoricalEnv( FileSystemFactory: &fixedFileSystemFactory{fs: snapshotFS}, }, }) - historicalEnv.SetPermissiveConfigParsing(env.PermissiveConfigParsing()) + historicalEnv.SetPermissiveConfigParsing(permissive) return historicalEnv, nil } diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go index f7b6795a7..7c82a2e07 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go @@ -302,6 +302,38 @@ path = "unavailable/curl.spec" assert.Equal(t, "abcdef1234567", project.comparisonInputs["rust-asyncband"].SourceIdentity) } +func TestLoadHistoricalProject_IgnoresUnknownHistoricalFields(t *testing.T) { + rootConfig := []byte(` +[project] +default-distro = { name = "testdistro", version = "1.0" } +rendered-specs-dir = "SPECS" + +[distros.testdistro] +description = "Test distro" +[distros.testdistro.versions."1.0"] +release-ver = "1.0" + +[test-suites.legacy] +description = "Removed config table" + +[components.curl.spec] +type = "upstream" +upstream-commit = "abcdef1234567" +`) + repo, hashes := testRepoWithCommits(t, []testRepoCommit{ + {files: map[string][]byte{"azldev.toml": rootConfig}}, + }) + tree, err := resolveTree(repo, hashes[0]) + require.NoError(t, err) + + testEnv := testutils.NewTestEnvWithoutLockfile(t) + require.False(t, testEnv.Env.PermissiveConfigParsing()) + + project, err := loadHistoricalProject(testEnv.Env, tree, ".") + require.NoError(t, err) + assert.Equal(t, "abcdef1234567", project.components["curl"].Spec.UpstreamCommit) +} + func TestSelectHistoricalComponentNames_UsesBothRefs(t *testing.T) { fromProject := &historicalProject{ components: map[string]projectconfig.ComponentConfig{ From 524413530d014cedaac84a7a8ea2f0d1525d5d9c Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 06:31:36 +0000 Subject: [PATCH 62/70] fix(component): resolve autorelease NEVR from dist-git history Lockless render validation queried existing and new source NEVRs with plain rpmspec, which expands an unprocessed `%autorelease -b N` to N. Builds instead filter the rendered directory's history and let rpmautospec count commits, so valid base bumps were rejected (e.g. SymCrypt, telegraf) and the check did not reflect the release actually built. For unprocessed %autorelease specs, reconstruct the filtered rendered history and run `rpmautospec process-distgit` before querying. The existing spec uses committed history; the new spec adds the prepared output as the next commit unless its tree is unchanged. Merge-free histories are replayed into a temporary repository that reads project objects via alternates and never writes to the project; histories with merges fall back to cloning and running `git filter-repo --subdirectory-filter`. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8b620ba1-9249-46b7-bfb4-78616d6f1ec1 --- docs/user/how-to/preview-without-lockfile.md | 18 +- .../azldev/cmds/component/render_history.go | 338 ++++++++++++++++++ .../component/render_history_internal_test.go | 139 +++++++ .../azldev/cmds/component/render_lockless.go | 93 ++++- .../component/render_output_internal_test.go | 152 +++++++- 5 files changed, 724 insertions(+), 16 deletions(-) create mode 100644 internal/app/azldev/cmds/component/render_history.go create mode 100644 internal/app/azldev/cmds/component/render_history_internal_test.go diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index ac99de9cc..725ef46e3 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -460,12 +460,22 @@ the working checkout retains its Git metadata until staging cleanup. After comparison and the final release bump, but immediately before removing existing output, lockless rendering queries the existing and new source-package NEVRs with host `rpmspec`. Queries expand each spec's macros using its own dist-git -directory. The package name must match. When the final new spec uses +directory. A spec whose `Release` uses `%autorelease` without having been processed +by rpmautospec (no `## RPMAUTOSPEC` header) is instead resolved as lockless builds +resolve it: against the component's filtered rendered-directory history, as +produced by `git filter-repo --subdirectory-filter`, using `rpmautospec +process-distgit` before the `rpmspec` query. The existing spec uses the committed +history; the new spec adds the prepared output as the next commit, unless its tree +is unchanged. For merge-free project history, the filtered history is replayed in a +temporary repository that reads project objects through Git alternates and never +writes to the project repository; histories with merges are cloned and filtered +with `git filter-repo`, which must then be installed. Plain `rpmspec` would expand +`%autorelease -b N` to `N`, ignoring the commits that raise the built release. +The package name must match. When the final new spec uses `%autorelease`, its new epoch/version/release must be equal to or higher than the existing value under RPM version ordering. Non-autorelease specs still require -a strictly higher value. Autorelease permits equality because a subsequent Git -commit can advance its Git-derived release; the query does not calculate that -future release. Lower NEVRs, and equal non-autorelease NEVRs, fail with both values +a strictly higher value. Autorelease permits equality because unchanged output +creates no new commit. Lower NEVRs, and equal non-autorelease NEVRs, fail with both values in the error message; query failures also abort. These errors return a failing exit status even without `--fail-on-error`, and leave existing output untouched. First renders with no existing directory skip the check. diff --git a/internal/app/azldev/cmds/component/render_history.go b/internal/app/azldev/cmds/component/render_history.go new file mode 100644 index 000000000..5712e1cd2 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_history.go @@ -0,0 +1,338 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "bytes" + "errors" + "fmt" + "os/exec" + "path/filepath" + "slices" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" +) + +// renderNEVRHistory resolves %autorelease specs against the component's +// filtered dist-git history, which is the history lockless builds use. +type renderNEVRHistory interface { + // Committed returns the NEVR of the committed rendered output. + Committed(filename string) (*rpm.SpecInfo, error) + // Rendered returns the NEVR the prepared output would have once committed. + Rendered(preparedDir, filename string) (*rpm.SpecInfo, error) + Remove() error +} + +//nolint:gochecknoglobals // Replaced by isolated NEVR tests that do not run git or rpmautospec. +var openRenderNEVRHistory = func(env *azldev.Env, parent, outputDir string) (renderNEVRHistory, error) { + return newRenderHistoryRepo(env, parent, outputDir) +} + +const renderHistoryProcessedSpec = "azldev-processed.spec" + +type renderHistoryRepo struct { + env *azldev.Env + dir string + tip string +} + +func newRenderHistoryRepo(env *azldev.Env, parent, outputDir string) (repo *renderHistoryRepo, err error) { + if err := prereqs.RequireExecutable(env, "git", nil); err != nil { + return nil, fmt.Errorf("lockless render NEVR validation requires 'git':\n%w", err) + } + + repoRoot, err := git.RunInDir(env, env, env.ProjectDir(), "rev-parse", "--show-toplevel") + if err != nil { + return nil, fmt.Errorf("finding project repository:\n%w", err) + } + + subdir, err := projectSpecRepoRelPath(env, repoRoot, outputDir) + if err != nil { + return nil, err + } + + dir, err := fileutils.MkdirTemp(env.FS(), parent, "azldev-render-history-") + if err != nil { + return nil, fmt.Errorf("creating dist-git history directory:\n%w", err) + } + + repo = &renderHistoryRepo{env: env, dir: dir} + + defer func() { + if err != nil { + fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), dir, &err) + } + }() + + merge, err := git.RunInDir(env, env, repoRoot, "rev-list", "--merges", "-n1", "HEAD") + if err != nil { + return nil, fmt.Errorf("checking project history for merges:\n%w", err) + } + + if merge != "" { + err = repo.filter(repoRoot, filepath.ToSlash(subdir)) + } else { + err = repo.replay(repoRoot, filepath.ToSlash(subdir)) + } + + if err != nil { + return nil, err + } + + return repo, nil +} + +// replay rebuilds the component's 'git filter-repo --subdirectory-filter' +// history without rewriting the whole project. It requires merge-free history, +// where the path-limited commit list matches the commits filter-repo keeps. +// The repository borrows project objects read-only through alternates. +func (r *renderHistoryRepo) replay(repoRoot, subdir string) error { + format, err := git.RunInDir(r.env, r.env, repoRoot, "rev-parse", "--show-object-format") + if err != nil { + return fmt.Errorf("reading project object format:\n%w", err) + } + + objects, err := git.RunInDir(r.env, r.env, repoRoot, "rev-parse", "--path-format=absolute", "--git-path", "objects") + if err != nil { + return fmt.Errorf("finding project objects:\n%w", err) + } + + if _, err := git.RunInDir(r.env, r.env, r.dir, "init", "-q", "--object-format="+format); err != nil { + return fmt.Errorf("creating dist-git history repository:\n%w", err) + } + + if err := fileutils.WriteFile(r.env.FS(), filepath.Join(r.dir, ".git", "objects", "info", "alternates"), + []byte(objects+"\n"), fileperms.PublicFile); err != nil { + return fmt.Errorf("sharing project objects:\n%w", err) + } + + if _, err := r.git("config", "gc.auto", "0"); err != nil { + return fmt.Errorf("disabling history repository maintenance:\n%w", err) + } + + list, err := git.RunInDir(r.env, r.env, repoRoot, "--literal-pathspecs", + "rev-list", "--reverse", "HEAD", "--", subdir) + if err != nil { + return fmt.Errorf("listing component dist-git commits:\n%w", err) + } + + for _, commit := range strings.Fields(list) { + if r.tip, err = r.replayCommit(commit, subdir); err != nil { + return fmt.Errorf("replaying commit %#q:\n%w", commit, err) + } + } + + if r.tip == "" { + return nil + } + + if _, err := r.git("update-ref", "HEAD", r.tip); err != nil { + return fmt.Errorf("updating dist-git history branch:\n%w", err) + } + + return nil +} + +func (r *renderHistoryRepo) replayCommit(commit, subdir string) (string, error) { + raw, err := r.git("cat-file", "commit", commit) + if err != nil { + return "", fmt.Errorf("reading commit:\n%w", err) + } + + tree, err := r.componentTree(commit, subdir) + if err != nil { + return "", err + } + + path := filepath.Join(r.dir, ".git", "azldev-commit") + if err := fileutils.WriteFile(r.env.FS(), path, []byte(rewriteRenderHistoryCommit(raw, tree, r.tip)), + fileperms.PublicFile); err != nil { + return "", fmt.Errorf("writing rewritten commit:\n%w", err) + } + + hash, err := r.git("hash-object", "-t", "commit", "-w", "--", path) + if err != nil { + return "", fmt.Errorf("writing rewritten commit:\n%w", err) + } + + return hash, nil +} + +func (r *renderHistoryRepo) componentTree(commit, subdir string) (string, error) { + entry, err := r.git("ls-tree", "--full-tree", commit, "--", subdir) + if err != nil { + return "", fmt.Errorf("reading component tree:\n%w", err) + } + + if fields := strings.Fields(entry); len(fields) >= 3 && fields[1] == "tree" { + return fields[2], nil + } + + // filter-repo keeps a deletion of the directory as an empty-tree commit. + tree, err := r.git("hash-object", "-t", "tree", "-w", "--stdin") + if err != nil { + return "", fmt.Errorf("writing empty tree:\n%w", err) + } + + return tree, nil +} + +// rewriteRenderHistoryCommit replaces a raw commit's tree and parent and drops +// signatures, as filter-repo does, keeping all other headers and the message. +func rewriteRenderHistoryCommit(raw, tree, parent string) string { + headers, message, _ := strings.Cut(raw, "\n\n") + rewritten := []string{"tree " + tree} + + if parent != "" { + rewritten = append(rewritten, "parent "+parent) + } + + skipping := false + + for _, line := range strings.Split(headers, "\n") { + if !strings.HasPrefix(line, " ") { + name, _, _ := strings.Cut(line, " ") + skipping = slices.Contains([]string{"tree", "parent", "gpgsig", "gpgsig-sha256", "mergetag"}, name) + } + + if !skipping { + rewritten = append(rewritten, line) + } + } + + return strings.Join(rewritten, "\n") + "\n\n" + message + "\n" +} + +// filter uses the same filtering lockless builds use, for histories with merges. +func (r *renderHistoryRepo) filter(repoRoot, subdir string) error { + if err := requireGitFilterRepo(r.env); err != nil { + return err + } + + if _, err := git.RunInDir(r.env, r.env, repoRoot, + "clone", "-q", "--single-branch", "--no-local", "--no-checkout", "--", repoRoot, r.dir); err != nil { + return fmt.Errorf("cloning project repository:\n%w", err) + } + + if _, err := r.git("filter-repo", "--force", "--subdirectory-filter", subdir); err != nil { + return fmt.Errorf("filtering component dist-git history:\n%w", err) + } + + // An empty filtered history leaves no HEAD commit; treat it as no history. + if tip, err := r.git("rev-parse", "--verify", "-q", "HEAD"); err == nil { + r.tip = tip + } + + return nil +} + +func (r *renderHistoryRepo) Committed(filename string) (*rpm.SpecInfo, error) { + if r.tip == "" { + return nil, errors.New("no committed dist-git history") + } + + return r.query(filename) +} + +func (r *renderHistoryRepo) Rendered(preparedDir, filename string) (*rpm.SpecInfo, error) { + // Stage the prepared output (git skips its '.git') as the next render commit. + if _, err := r.git("--work-tree="+preparedDir, "add", "-A"); err != nil { + return nil, fmt.Errorf("staging prepared output:\n%w", err) + } + + tree, err := r.git("write-tree") + if err != nil { + return nil, fmt.Errorf("writing prepared output tree:\n%w", err) + } + + previous := "" + if r.tip != "" { + if previous, err = r.git("rev-parse", r.tip+"^{tree}"); err != nil { + return nil, fmt.Errorf("reading committed tree:\n%w", err) + } + } + + // Unchanged output produces no render commit, so the committed history stays current. + if tree != previous { + args := []string{ + "-c", "user.name=azldev", "-c", "user.email=azldev@localhost", + "commit-tree", "--no-gpg-sign", "-m", "azldev render NEVR validation", + } + if r.tip != "" { + args = append(args, "-p", r.tip) + } + + commit, err := r.git(append(args, tree)...) + if err != nil { + return nil, fmt.Errorf("committing prepared output:\n%w", err) + } + + if _, err := r.git("update-ref", "HEAD", commit); err != nil { + return nil, fmt.Errorf("updating dist-git history branch:\n%w", err) + } + + r.tip = commit + } + + return r.query(filename) +} + +// query checks out the history tip and resolves its spec as the mock rpmautospec plugin does. +func (r *renderHistoryRepo) query(filename string) (*rpm.SpecInfo, error) { + // Rebuild the index so every file is written, not only those differing from staged stat data. + for _, args := range [][]string{ + {"read-tree", "--empty"}, {"clean", "-q", "-f", "-d", "-x"}, {"read-tree", "--reset", "-u", r.tip}, + } { + if _, err := r.git(args...); err != nil { + return nil, fmt.Errorf("checking out dist-git history:\n%w", err) + } + } + + if err := prereqs.RequireExecutable(r.env, "rpmautospec", nil); err != nil { + return nil, fmt.Errorf("lockless render NEVR validation requires 'rpmautospec':\n%w", err) + } + + processed := filepath.Join(r.dir, ".git", renderHistoryProcessedSpec) + + raw := exec.CommandContext(r.env, "rpmautospec", "process-distgit", filename, processed) + raw.Dir = r.dir + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + command, err := r.env.Command(raw) + if err != nil { + return nil, fmt.Errorf("creating rpmautospec command:\n%w", err) + } + + if _, err := command.RunAndGetOutput(r.env); err != nil { + return nil, fmt.Errorf("processing %#q with rpmautospec (stderr: %#q):\n%w", filename, stderr.String(), err) + } + + return queryRenderSpecNEVR(r.env, processed, r.dir) +} + +func (r *renderHistoryRepo) Remove() error { + if err := r.env.FS().RemoveAll(r.dir); err != nil { + return fmt.Errorf("removing dist-git history directory %#q:\n%w", r.dir, err) + } + + return nil +} + +func (r *renderHistoryRepo) git(args ...string) (string, error) { + output, err := git.RunInDir(r.env, r.env, r.dir, args...) + if err != nil { + return "", fmt.Errorf("running git in dist-git history:\n%w", err) + } + + return output, nil +} diff --git a/internal/app/azldev/cmds/component/render_history_internal_test.go b/internal/app/azldev/cmds/component/render_history_internal_test.go new file mode 100644 index 000000000..64551bd12 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_history_internal_test.go @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const renderHistoryLogFormat = "--format=%T %an <%ae> %ad %cn <%ce> %cd %s" + +func runTestGit(t *testing.T, dir string, args ...string) string { + t.Helper() + + cmd := exec.CommandContext(t.Context(), "git", append([]string{"-C", dir}, args...)...) + cmd.Env = append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=Author", "GIT_AUTHOR_EMAIL=author@example.com", + "GIT_COMMITTER_NAME=Committer", "GIT_COMMITTER_EMAIL=committer@example.com") + + output, err := cmd.CombinedOutput() + require.NoError(t, err, string(output)) + + return strings.TrimSpace(string(output)) +} + +func writeTestGitFile(t *testing.T, repo, path, content string) { + t.Helper() + + path = filepath.Join(repo, path) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) +} + +func commitTestGit(t *testing.T, repo, subject string) { + t.Helper() + + runTestGit(t, repo, "add", "-A") + runTestGit(t, repo, "commit", "-q", "--allow-empty", "-m", subject, "-m", "body for "+subject) +} + +func TestRenderHistoryRepoMatchesFilteredHistory(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git is not installed") + } + + for _, merge := range []bool{false, true} { + t.Run(map[bool]string{false: "replay", true: "filter fallback for merges"}[merge], func(t *testing.T) { + _, filterErr := exec.LookPath("git-filter-repo") + if merge && filterErr != nil { + t.Skip("git-filter-repo is not installed") + } + + root := t.TempDir() + repo := filepath.Join(root, "project") + runTestGit(t, root, "init", "-q", "-b", "main", repo) + + writeTestGitFile(t, repo, "specs/c/curl/curl.spec", "v1") + writeTestGitFile(t, repo, "specs/c/curl-other/curl.spec", "other") + commitTestGit(t, repo, "add curl") + writeTestGitFile(t, repo, "base/comps/curl.toml", "config") + commitTestGit(t, repo, "configure curl") + + if merge { + runTestGit(t, repo, "checkout", "-q", "-b", "side") + writeTestGitFile(t, repo, "specs/c/curl/side.patch", "side") + commitTestGit(t, repo, "side change") + runTestGit(t, repo, "checkout", "-q", "main") + writeTestGitFile(t, repo, "README", "readme") + commitTestGit(t, repo, "main change") + runTestGit(t, repo, "merge", "-q", "--no-ff", "-m", "merge side", "side") + } + + writeTestGitFile(t, repo, "specs/c/curl/curl.spec", "v2") + commitTestGit(t, repo, "update curl") + runTestGit(t, repo, "rm", "-q", "-r", "specs/c/curl") + commitTestGit(t, repo, "remove curl") + writeTestGitFile(t, repo, "specs/c/curl/curl.spec", "v3") + commitTestGit(t, repo, "restore curl") + writeTestGitFile(t, repo, "specs/c/curl-other/curl.spec", "other 2") + commitTestGit(t, repo, "update other") + + testEnv := testutils.NewTestEnv(t) + dryRun := azldev.NewAppDryRunnable(false) + cmdFactory, err := azldev.DefaultCmdFactory(dryRun, testEnv.EventListener) + require.NoError(t, err) + + env := azldev.NewEnv(t.Context(), azldev.EnvOptions{ + ProjectDir: repo, Config: testEnv.Config, DryRunnable: dryRun, EventListener: testEnv.EventListener, + Interfaces: azldev.SystemInterfaces{ + CmdFactory: cmdFactory, FileSystemFactory: azldev.DefaultFileSystemFactory(), + OSEnvFactory: azldev.DefaultOSEnvFactory(), + }, + }) + + staging := filepath.Join(root, "staging") + require.NoError(t, os.Mkdir(staging, 0o755)) + + history, err := newRenderHistoryRepo(env, staging, filepath.Join(repo, "specs/c/curl")) + require.NoError(t, err) + + assert.Equal(t, runTestGit(t, repo, "rev-parse", "HEAD:specs/c/curl"), + runTestGit(t, history.dir, "rev-parse", "HEAD^{tree}")) + + replayed := runTestGit(t, history.dir, "log", renderHistoryLogFormat) + if !merge { + assert.Equal(t, []string{"restore curl", "remove curl", "update curl", "add curl"}, + strings.Split(runTestGit(t, history.dir, "log", "--format=%s"), "\n")) + assert.Equal(t, "body for add curl", + runTestGit(t, history.dir, "log", "-1", "--format=%b", "HEAD~3")) + } + + if filterErr == nil { + filtered := filepath.Join(root, "filtered") + runTestGit(t, root, "clone", "-q", "--single-branch", "--no-local", "--", repo, filtered) + runTestGit(t, filtered, "filter-repo", "--force", "--subdirectory-filter", "specs/c/curl") + assert.Equal(t, runTestGit(t, filtered, "log", renderHistoryLogFormat), replayed) + } + + // The project repository is only read; the replay writes its own objects. + assert.Empty(t, runTestGit(t, repo, "status", "--porcelain")) + runTestGit(t, repo, "fsck", "--no-progress") + + require.NoError(t, history.Remove()) + + entries, err := os.ReadDir(staging) + require.NoError(t, err) + assert.Empty(t, entries) + }) + } +} diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index 2e6bca9e6..b433dd89e 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -1024,17 +1024,26 @@ func locklessSpecUsesAutorelease(env *azldev.Env, path string) (bool, error) { return sources.ReleaseUsesAutorelease(release), nil } +// validateRenderNEVR rejects replacing rendered output with a lower NEVR. +// Unprocessed %autorelease specs are resolved against the component's filtered +// dist-git history, as lockless builds resolve them, instead of plain rpmspec. func validateRenderNEVR(env *azldev.Env, preparedDir string, prep *preparedComponent) error { - existing, err := queryRenderSpecNEVR(env, filepath.Join(prep.compOutputDir, prep.specFilename)) + existingPath := filepath.Join(prep.compOutputDir, prep.specFilename) + newPath := filepath.Join(preparedDir, prep.specFilename) + + existingHistory, err := renderSpecNeedsHistory(env, existingPath) if err != nil { - return fmt.Errorf("%w: querying existing NEVR:\n%w", errRenderNEVRValidation, err) + return fmt.Errorf("%w: checking existing spec autorelease:\n%w", errRenderNEVRValidation, err) } - newPath := filepath.Join(preparedDir, prep.specFilename) + newHistory, err := renderSpecNeedsHistory(env, newPath) + if err != nil { + return fmt.Errorf("%w: checking new spec autorelease:\n%w", errRenderNEVRValidation, err) + } - rendered, err := queryRenderSpecNEVR(env, newPath) + existing, rendered, err := queryRenderNEVRs(env, preparedDir, prep, existingHistory, newHistory) if err != nil { - return fmt.Errorf("%w: querying new NEVR:\n%w", errRenderNEVRValidation, err) + return err } autorelease, err := locklessSpecUsesAutorelease(env, newPath) @@ -1057,12 +1066,82 @@ func validateRenderNEVR(env *azldev.Env, preparedDir string, prep *preparedCompo return nil } -func queryRenderSpecNEVR(env *azldev.Env, path string) (*rpm.SpecInfo, error) { +func queryRenderNEVRs( + env *azldev.Env, preparedDir string, prep *preparedComponent, existingHistory, newHistory bool, +) (existing, rendered *rpm.SpecInfo, err error) { + var history renderNEVRHistory + + if existingHistory || newHistory { + history, err = openRenderNEVRHistory(env, filepath.Dir(preparedDir), prep.compOutputDir) + if err != nil { + return nil, nil, fmt.Errorf("%w: reading dist-git history:\n%w", errRenderNEVRValidation, err) + } + + defer func() { + if removeErr := history.Remove(); removeErr != nil && err == nil { + err = fmt.Errorf("%w:\n%w", errRenderNEVRValidation, removeErr) + } + }() + } + + if existingHistory { + existing, err = history.Committed(prep.specFilename) + } else { + existing, err = queryRenderSpecNEVR(env, filepath.Join(prep.compOutputDir, prep.specFilename), + prep.compOutputDir) + } + + if err != nil { + return nil, nil, fmt.Errorf("%w: querying existing NEVR:\n%w", errRenderNEVRValidation, err) + } + + if newHistory { + rendered, err = history.Rendered(preparedDir, prep.specFilename) + } else { + rendered, err = queryRenderSpecNEVR(env, filepath.Join(preparedDir, prep.specFilename), preparedDir) + } + + if err != nil { + return nil, nil, fmt.Errorf("%w: querying new NEVR:\n%w", errRenderNEVRValidation, err) + } + + return existing, rendered, nil +} + +// renderSpecNeedsHistory reports whether a spec uses %autorelease without +// having been processed by rpmautospec, so its release depends on history. +func renderSpecNeedsHistory(env *azldev.Env, path string) (bool, error) { + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return false, fmt.Errorf("reading spec %#q:\n%w", path, err) + } + + if renderedAutoreleaseMarker.Match(content) { + return false, nil + } + + opened, err := spec.OpenSpec(bytes.NewReader(content)) + if err != nil { + return false, fmt.Errorf("parsing spec %#q:\n%w", path, err) + } + + release, err := opened.GetLastTag("", "Release") + if errors.Is(err, spec.ErrNoSuchTag) { + return false, nil + } + + if err != nil { + return false, fmt.Errorf("reading Release from %#q:\n%w", path, err) + } + + return sources.ReleaseUsesAutorelease(release), nil +} + +func queryRenderSpecNEVR(env *azldev.Env, path, dir string) (*rpm.SpecInfo, error) { if err := prereqs.RequireExecutable(env, "rpmspec", nil); err != nil { return nil, fmt.Errorf("lockless render NEVR validation requires 'rpmspec':\n%w", err) } - dir := filepath.Dir(path) raw := exec.CommandContext(env, "rpmspec", "-q", "--srpm", "--define", "_sourcedir "+dir, "--define", "_specdir "+dir, "--queryformat", rpm.SrpmQueryFormat, path) diff --git a/internal/app/azldev/cmds/component/render_output_internal_test.go b/internal/app/azldev/cmds/component/render_output_internal_test.go index 0ccf0efa1..135249a96 100644 --- a/internal/app/azldev/cmds/component/render_output_internal_test.go +++ b/internal/app/azldev/cmds/component/render_output_internal_test.go @@ -12,6 +12,7 @@ import ( "syscall" "testing" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" @@ -31,6 +32,7 @@ import ( const ( testRenderOutputDir = "/project/SPECS/c/curl" + testRenderStagingDir = "/staging/curl" renderTestDiffCommand = "diff" renderTestResetCommand = "reset" renderTestCheckoutCommand = "checkout" @@ -97,6 +99,9 @@ func TestLocklessRenderNEVRBeforeReplacement(t *testing.T) { newName = testCase.newName } + history := &fakeRenderNEVRHistory{rendered: &rpm.SpecInfo{Name: newName, Version: *rendered}} + opened := stubRenderNEVRHistory(t, history) + queries := 0 env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { queries++ @@ -108,7 +113,7 @@ func TestLocklessRenderNEVRBeforeReplacement(t *testing.T) { name, version := "curl", existing if queries == 2 { - dir, name, version = "/staging/curl", newName, rendered + dir, name, version = testRenderStagingDir, newName, rendered } assert.Equal(t, dir, cmd.Dir) @@ -126,7 +131,7 @@ func TestLocklessRenderNEVRBeforeReplacement(t *testing.T) { "\nversion=" + version.Version() + "\nrelease=" + version.Release() + "\n", nil } - err = moveRenderedOutput(env.Env, "/staging/curl", prep, true) + err = moveRenderedOutput(env.Env, testRenderStagingDir, prep, true) if testCase.reject { require.ErrorIs(t, err, errRenderNEVRValidation) assertOriginalRenderOutput(t, env.TestFS, true) @@ -160,8 +165,15 @@ func TestLocklessRenderNEVRBeforeReplacement(t *testing.T) { assert.Zero(t, queries) case testCase.queryFails: assert.Equal(t, 1, queries) + case testCase.rawRelease != "": + // Only the existing spec uses rpmspec; history resolves the new %autorelease. + assert.Equal(t, 1, queries) + assert.Equal(t, 1, *opened) + assert.Equal(t, testRenderStagingDir, history.preparedDir) + assert.True(t, history.removed) default: assert.Equal(t, 2, queries) + assert.Zero(t, *opened) } }) } @@ -184,7 +196,7 @@ func TestRenderNEVRSkipsFirstAndLockedRenders(t *testing.T) { return "", nil } - require.NoError(t, moveRenderedOutput(env.Env, "/staging/curl", &preparedComponent{ + require.NoError(t, moveRenderedOutput(env.Env, testRenderStagingDir, &preparedComponent{ lockless: !existing, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, }, true)) }) @@ -238,15 +250,79 @@ func TestMoveRenderedOutput_LockModes(t *testing.T) { } } +type fakeRenderNEVRHistory struct { + committed, rendered *rpm.SpecInfo + preparedDir string + removed bool +} + +func (h *fakeRenderNEVRHistory) Committed(string) (*rpm.SpecInfo, error) { + if h.committed == nil { + return nil, errors.New("unexpected committed history query") + } + + return h.committed, nil +} + +func (h *fakeRenderNEVRHistory) Rendered(preparedDir, _ string) (*rpm.SpecInfo, error) { + h.preparedDir = preparedDir + + return h.rendered, nil +} + +func (h *fakeRenderNEVRHistory) Remove() error { + h.removed = true + + return nil +} + +// stubRenderNEVRHistory replaces dist-git history resolution; callers must not run in parallel. +func stubRenderNEVRHistory(t *testing.T, history renderNEVRHistory) *int { + t.Helper() + + opened := 0 + original := openRenderNEVRHistory + openRenderNEVRHistory = func(_ *azldev.Env, parent, outputDir string) (renderNEVRHistory, error) { + opened++ + + assert.Equal(t, testRenderOutputDir, outputDir) + assert.Equal(t, "/staging", parent) + + return history, nil + } + + t.Cleanup(func() { openRenderNEVRHistory = original }) + + return &opened +} + +func testSpecInfo(t *testing.T, evr string) *rpm.SpecInfo { + t.Helper() + + version, err := rpm.NewVersion(evr) + require.NoError(t, err) + + return &rpm.SpecInfo{Name: "curl", Version: *version} +} + func stubIncreasingRenderNEVR(t *testing.T, env *testutils.TestEnv) { t.Helper() + originalHistory := openRenderNEVRHistory + openRenderNEVRHistory = func(*azldev.Env, string, string) (renderNEVRHistory, error) { + return &fakeRenderNEVRHistory{ + committed: testSpecInfo(t, "1-1"), rendered: testSpecInfo(t, "1-2"), + }, nil + } + + t.Cleanup(func() { openRenderNEVRHistory = originalHistory }) + env.CmdFactory.RegisterCommandInSearchPath("rpmspec") original := env.CmdFactory.RunAndGetOutputHandler env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { if cmd.Args[0] == "rpmspec" { release := "1" - if cmd.Dir == "/staging/curl" { + if cmd.Dir == testRenderStagingDir { release = "2" } @@ -280,7 +356,7 @@ func setupRenderOutput(t *testing.T, fileSystem afero.Fs, existing bool) { for name, content := range map[string]string{ "curl.spec": "Name: curl\nVersion: 1\nRelease: 1\n", "sidecar": "new sidecar", ".git/HEAD": "upstream", } { - require.NoError(t, fileutils.WriteFile(fileSystem, filepath.Join("/staging/curl", name), + require.NoError(t, fileutils.WriteFile(fileSystem, filepath.Join(testRenderStagingDir, name), []byte(content), fileperms.PublicFile)) } @@ -556,3 +632,69 @@ func (fs *renderOutputFaultFS) RemoveAll(path string) error { return fs.Fs.RemoveAll(path) } + +func TestLocklessRenderNEVRResolvesAutoreleaseHistory(t *testing.T) { + const processedSpec = "## RPMAUTOSPEC: autorelease, autochangelog\nName: curl\nVersion: 1\nRelease: 7\n" + + for _, testCase := range []struct { + name, existingSpec, committed, rendered string + reject bool + }{ + // rpmspec without history expands '-b 6' to 6, below the processed 7. + {name: "history raises base above processed release", existingSpec: processedSpec, rendered: "1-8"}, + {name: "history release below processed release", existingSpec: processedSpec, rendered: "1-6", reject: true}, + { + name: "unprocessed existing spec uses history", existingSpec: renderTestSpec("1", "%autorelease -b 2", ""), + committed: "1-5", rendered: "1-6", + }, + { + name: "unprocessed existing release above new", existingSpec: renderTestSpec("1", "%autorelease -b 2", ""), + committed: "1-7", rendered: "1-6", reject: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, true) + env.CmdFactory.RegisterCommandInSearchPath("rpmspec") + + for _, root := range []string{testRenderOutputDir, "/before"} { + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(root, "curl.spec"), + []byte(testCase.existingSpec), fileperms.PublicFile)) + } + + require.NoError(t, fileutils.WriteFile(env.TestFS, "/staging/curl/curl.spec", + []byte(renderTestSpec("1", "%autorelease -b 6", "")), fileperms.PublicFile)) + + history := &fakeRenderNEVRHistory{rendered: testSpecInfo(t, testCase.rendered)} + if testCase.committed != "" { + history.committed = testSpecInfo(t, testCase.committed) + } + + stubRenderNEVRHistory(t, history) + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, processedSpec, testCase.existingSpec, "only processed specs use plain rpmspec") + assert.Equal(t, testRenderOutputDir, cmd.Dir) + + release := "7" + if cmd.Dir == testRenderStagingDir { + release = "6" + } + + return "name=curl\nepoch=0\nversion=1\nrelease=" + release + "\n", nil + } + + err := moveRenderedOutput(env.Env, testRenderStagingDir, &preparedComponent{ + lockless: true, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, + }, true) + if testCase.reject { + require.ErrorIs(t, err, errRenderNEVRValidation) + assertOriginalRenderOutput(t, env.TestFS, true) + } else { + require.NoError(t, err) + } + + assert.True(t, history.removed) + }) + } +} From efc49492435264618f4e6e888caeecf726ebe8f3 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Sun, 8 Nov 2026 16:00:45 +0000 Subject: [PATCH 63/70] fix(component): ignore %changelog Release lines in lockless bumps rpmdev-bumpspec bumps every line starting with 'Release:', including text inside %changelog, such as a quoted spec snippet in a previous entry. Lockless renders then rewrote that historical changelog text. Before running rpmdev-bumpspec for a lockless component, temporarily mask %changelog lines that start with 'Release:' and restore them afterward, also restoring the original spec if the bump fails. Specs without such lines are bumped exactly as before, and locked components are unaffected because they use a separate bumpspec path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 16503351-1bf3-4693-96f1-26283e4fc641 --- docs/user/how-to/preview-without-lockfile.md | 4 +- .../azldev/cmds/component/render_lockless.go | 81 +++++++++++++++++++ .../render_lockless_internal_test.go | 48 +++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 725ef46e3..89b195858 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -413,7 +413,9 @@ For upstream non-autorelease specs, a temporary process-local RPM wrapper sets `_sourcedir` and `_specdir` to the spec's directory so sidecar macros resolve correctly. Rendering preserves your `HOME`, packager settings, locale, and timezone; it does not edit your `.rpmmacros`. The wrapper is removed after the -invocation, including when bumpspec fails. +invocation, including when bumpspec fails. `Release:` lines inside `%changelog` +text (for example, quoted spec snippets) are temporarily masked so bumpspec +changes only real `Release` tags; they are restored unchanged afterward. When `--changelog-comment` is omitted, the prompt depends on how the message is used: diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go index b433dd89e..4758b371d 100644 --- a/internal/app/azldev/cmds/component/render_lockless.go +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -1227,6 +1227,80 @@ func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate return false, fmt.Errorf("lockless upstream rendering requires 'rpmdev-bumpspec'; install 'rpmdevtools':\n%w", err) } + masked, err := maskChangelogReleaseLines(opened, before) + if err != nil { + return false, err + } + + if masked == nil { + return runLocklessBumpspec(env, path, update, before) + } + + if err := fileutils.WriteFile(env.FS(), path, masked, fileperms.PublicFile); err != nil { + return false, fmt.Errorf("masking %%changelog Release lines before release bump:\n%w", err) + } + + changed, err = runLocklessBumpspec(env, path, update, before) + if err != nil { + if restoreErr := fileutils.WriteFile(env.FS(), path, before, fileperms.PublicFile); restoreErr != nil { + err = errors.Join(err, fmt.Errorf("restoring spec after failed release bump:\n%w", restoreErr)) + } + + return false, err + } + + return changed, nil +} + +// changelogReleaseMask prefixes '%changelog' lines that rpmdev-bumpspec would otherwise +// treat as Release tags; rpmdev-bumpspec does not distinguish spec sections. +const changelogReleaseMask = "azldev-masked-changelog-line:" + +var changelogReleaseLinePattern = regexp.MustCompile(`(?i)^release\s*:`) + +// maskChangelogReleaseLines returns the spec with '%changelog' lines starting with 'Release:' +// masked, or nil when there are no such lines. +func maskChangelogReleaseLines(opened *spec.Spec, original []byte) ([]byte, error) { + masked := false + + err := opened.Visit(func(ctx *spec.Context) error { + if ctx.Target.TargetType != spec.SectionLineTarget || ctx.RawLine == nil || + ctx.CurrentSection.SectType != spec.ChangelogSection || + !changelogReleaseLinePattern.MatchString(*ctx.RawLine) { + return nil + } + + ctx.ReplaceLine(changelogReleaseMask + *ctx.RawLine) + + masked = true + + return nil + }) + if err != nil { + return nil, fmt.Errorf("finding %%changelog Release lines:\n%w", err) + } + + if !masked { + return nil, nil + } + + if bytes.Contains(original, []byte(changelogReleaseMask)) { + return nil, fmt.Errorf("spec already contains reserved text %#q", changelogReleaseMask) + } + + var buffer bytes.Buffer + if err := opened.Serialize(&buffer); err != nil { + return nil, fmt.Errorf("masking %%changelog Release lines:\n%w", err) + } + + return buffer.Bytes(), nil +} + +// runLocklessBumpspec runs rpmdev-bumpspec, unmasks any masked '%changelog' lines, and +// reports whether the spec differs from before. +func runLocklessBumpspec( + env *azldev.Env, path string, update *locklessRenderUpdate, before []byte, +) (changed bool, err error) { raw := exec.CommandContext(env, sources.RPMDevBumpspecBinary, "-c", update.subject, path) raw.Dir = filepath.Dir(path) @@ -1257,6 +1331,13 @@ func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate return false, fmt.Errorf("reading spec after release bump:\n%w", err) } + if bytes.Contains(after, []byte(changelogReleaseMask)) { + after = bytes.ReplaceAll(after, []byte(changelogReleaseMask), nil) + if err := fileutils.WriteFile(env.FS(), path, after, fileperms.PublicFile); err != nil { + return false, fmt.Errorf("unmasking %%changelog Release lines after release bump:\n%w", err) + } + } + return !bytes.Equal(before, after), nil } diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index bf756eadf..d442ef747 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "slices" "strconv" "strings" @@ -1704,6 +1705,53 @@ func TestBumpLocklessSpec_Failures(t *testing.T) { } } +func TestBumpLocklessSpec_IgnoresChangelogReleaseLines(t *testing.T) { + const path = "/working/curl.spec" + + changelog := "* Previous\n- Example:\nRelease: %%{release_string}.2\nrelease : 3\n" + before := renderTestSpec("1", "1", changelog) + + for _, fail := range []bool{false, true} { + t.Run(strconv.FormatBool(fail), func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(before), fileperms.PublicFile)) + + // Mimic rpmdev-bumpspec, which bumps every line starting with 'Release:'. + env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + current, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + + bumped := regexp.MustCompile(`(?mi)^release\s*:.*$`).ReplaceAll(current, []byte("Release: BUMPED")) + require.NoError(t, fileutils.WriteFile(env.TestFS, path, bumped, fileperms.PublicFile)) + + if fail { + return errors.New("bumpspec failed") + } + + return nil + } + + changed, err := bumpLocklessSpec(env.Env, path, &locklessRenderUpdate{subject: "Render message"}) + + result, readErr := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, readErr) + + if fail { + require.ErrorContains(t, err, "bumpspec failed") + assert.False(t, changed) + assert.Equal(t, before, string(result)) + + return + } + + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, renderTestSpec("1", "BUMPED", changelog), string(result)) + }) + } +} + func TestBumpLocklessSpec_ProcessLocalRPMContext(t *testing.T) { for _, fail := range []bool{false, true} { t.Run(strconv.FormatBool(fail), func(t *testing.T) { From c2297fc999c781e24dd3ec911b129f14999e04dd Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 11 Nov 2026 15:19:43 +0000 Subject: [PATCH 64/70] fix(component): fetch configured sources for lockless builds Use the source manager to retrieve configured payloads before downloading remaining dist-git manifest entries from lookaside. Preserve download-origin fallback, custom generation, hash validation, and disable-origins policy without replaying overlays or replacing the committed rendered spec. Cover origin downloads with and without a manifest, disabled origins, and custom-provider dispatch. Update build guidance for both skill catalogs. Validation: mage fix all, mage unit, and mage all build/quality checks passed. Container scenarios were blocked by Docker socket permissions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83cf8df4-30c0-474c-8dca-e3b3a2e54873 --- docs/user/how-to/build-component.md | 3 +- docs/user/how-to/preview-without-lockfile.md | 8 ++- .../content/build-component.md.tmpl | 3 +- .../withoutlockfile/build-component.md.tmpl | 3 +- .../azldev/cmds/component/build_lockless.go | 23 ++++++- .../component/build_lockless_internal_test.go | 69 +++++++++++++++++++ 6 files changed, 100 insertions(+), 9 deletions(-) diff --git a/docs/user/how-to/build-component.md b/docs/user/how-to/build-component.md index e5b4a5d14..6bdc4226e 100644 --- a/docs/user/how-to/build-component.md +++ b/docs/user/how-to/build-component.md @@ -28,7 +28,8 @@ Built packages are placed in structured subdirectories under the project's outpu For components listed in `lockless-components.txt`, render and commit first. Build extracts the committed component dist-git history using `git-filter-repo`, -downloads lookaside sources, and builds the SRPM with Mock's `rpmautospec` plugin. +fetches configured source files (including download and custom origins), downloads +remaining manifest payloads from lookaside, and builds the SRPM with Mock's `rpmautospec` plugin. Install `git-filter-repo` and Mock 5.3 or newer. Uncommitted rendered edits are ignored. Lockless builds reject `--without-git` and `--rpmdev-bumpspec`; lock-file-based builds are unchanged. See the diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 89b195858..265af112b 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -110,9 +110,11 @@ Mock chroot, then reports its source NEVR, required files, and binary subpackage The temporary query repositories are removed on success and failure. Locked queries continue to read working-tree rendered files, including uncommitted edits. -`component build` downloads sources listed in the extracted `sources` manifest, -then builds the SRPM with Mock's `rpmautospec` plugin enabled and the extracted -repository passed as `--sources`. This requires Mock 5.3 or newer and +`component build` fetches configured source files through the normal source manager, +including custom generation and download-origin fallback unless `disable-origins` +is enabled. It then downloads any remaining payloads listed in the extracted +`sources` manifest from lookaside and builds the SRPM with Mock's `rpmautospec` +plugin enabled and the extracted repository passed as `--sources`. This requires Mock 5.3 or newer and rpmautospec available from the build distro's package repositories. Mock's rpmautospec plugin must be installed on the host as well; enabling an unavailable plugin is an error, even if Mock itself would only warn. diff --git a/internal/app/azldev/agentskill/content/build-component.md.tmpl b/internal/app/azldev/agentskill/content/build-component.md.tmpl index e318bcb88..6a1b8f14a 100644 --- a/internal/app/azldev/agentskill/content/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/build-component.md.tmpl @@ -24,7 +24,8 @@ progress — use it only for inner-loop builds you expect to succeed. ## The inner loop For lockless components, build uses committed rendered dist-git history extracted -with git-filter-repo; render and commit first. It downloads lookaside sources and +with git-filter-repo; render and commit first. It fetches configured sources through +the source manager (including download/custom origins), then remaining lookaside payloads, and builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` are rejected for lockless builds; `--no-check` is forwarded normally. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl index 3037d9519..6153e1264 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl @@ -24,7 +24,8 @@ progress — use it only for inner-loop builds you expect to succeed. ## The inner loop For lockless components, build uses committed rendered dist-git history extracted -with git-filter-repo; render and commit first. It downloads lookaside sources and +with git-filter-repo; render and commit first. It fetches configured sources through +the source manager (including download/custom origins), then remaining lookaside payloads, and builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` are rejected for lockless builds; `--no-check` is forwarded normally. diff --git a/internal/app/azldev/cmds/component/build_lockless.go b/internal/app/azldev/cmds/component/build_lockless.go index 52ca74f61..f8c065622 100644 --- a/internal/app/azldev/cmds/component/build_lockless.go +++ b/internal/app/azldev/cmds/component/build_lockless.go @@ -12,6 +12,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/componentbuilder" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/workdir" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" "github.com/microsoft/azure-linux-dev-tools/internal/utils/defers" @@ -38,7 +39,7 @@ func buildLocklessComponent( return results, err } - if err := downloadDistGitSources(env, name, dir); err != nil { + if err := downloadDistGitSources(env, component, dir); err != nil { return results, err } @@ -70,7 +71,23 @@ func buildLocklessComponent( options.NoCheck, options.LocalRepoWithPublishPath) } -func downloadDistGitSources(env *azldev.Env, name, dir string) error { +func downloadDistGitSources(env *azldev.Env, component components.Component, dir string) error { + if len(component.GetConfig().SourceFiles) > 0 { + distro, err := sourceproviders.ResolveDistro(env, component) + if err != nil { + return fmt.Errorf("resolving source distro:\n%w", err) + } + + manager, err := sourceproviders.NewSourceManager(env, distro) + if err != nil { + return fmt.Errorf("creating source manager:\n%w", err) + } + + if err := manager.FetchFiles(env, component, dir); err != nil { + return fmt.Errorf("fetching configured dist-git sources:\n%w", err) + } + } + path := filepath.Join(dir, "sources") exists, err := fileutils.Exists(env.FS(), path) @@ -104,7 +121,7 @@ func downloadDistGitSources(env *azldev.Env, name, dir string) error { if !exists { if err := downloadsources.DownloadSources(env, &downloadsources.DownloadSourcesOptions{ - Directory: dir, ComponentName: name, + Directory: dir, ComponentName: component.GetName(), }); err != nil { return fmt.Errorf("downloading dist-git sources:\n%w", err) } diff --git a/internal/app/azldev/cmds/component/build_lockless_internal_test.go b/internal/app/azldev/cmds/component/build_lockless_internal_test.go index 254dde40e..a0257c483 100644 --- a/internal/app/azldev/cmds/component/build_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/build_lockless_internal_test.go @@ -11,6 +11,7 @@ import ( "strings" "testing" + "github.com/h2non/gock" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/workdir" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" @@ -20,6 +21,74 @@ import ( "github.com/stretchr/testify/require" ) +func TestDownloadDistGitSources_ConfiguredOrigins(t *testing.T) { + const ( + filename = "extra-source.tar.gz" + payload = "hello world" + digest = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + ) + + for _, testCase := range []struct { + name string + manifest bool + disableOrigins bool + custom bool + wantError string + }{ + {name: "download with manifest", manifest: true}, + {name: "download without manifest"}, + {name: "origins disabled", manifest: true, disableOrigins: true, wantError: "disable-origins is enabled"}, + {name: "custom origin dispatched", manifest: true, custom: true, wantError: "generation script"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + distro := env.Config.Distros["test-distro"] + distro.LookasideBaseURI = "" + distro.DisableOrigins = testCase.disableOrigins + env.Config.Distros["test-distro"] = distro + + ref := projectconfig.SourceFileReference{ + Filename: filename, Hash: digest, HashType: fileutils.HashTypeSHA256, + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://origin.invalid/source"}, + } + if testCase.custom { + ref.Origin = projectconfig.Origin{Type: projectconfig.OriginTypeCustom, Script: "/project/generate.sh"} + } + + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", SourceFiles: []projectconfig.SourceFileReference{ref}, + } + dir := "/project/dist-git" + require.NoError(t, fileutils.MkdirAll(env.FS(), dir)) + + if testCase.manifest { + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(dir, "sources"), + []byte("SHA256 ("+filename+") = "+digest+"\n"), fileperms.PublicFile)) + } + + if testCase.wantError == "" { + gock.New("https://origin.invalid").Get("/source").Reply(200).BodyString(payload) + t.Cleanup(gock.OffAll) + } + + err := downloadDistGitSources(env.Env, resolveComponents(t, env, "curl")[0], dir) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + + return + } + + require.NoError(t, err) + require.True(t, gock.IsDone()) + + content, err := fileutils.ReadFile(env.FS(), filepath.Join(dir, filename)) + require.NoError(t, err) + require.Equal(t, payload, string(content)) + }) + } +} + func TestLocklessBuildRejectsSyntheticHistoryFlags(t *testing.T) { for _, options := range []ComponentBuildOptions{{WithoutGitRepo: true}, {RPMDevBumpspec: true}} { env := testutils.NewTestEnv(t) From a9668034307f8d7c4cec4ec405c6fe7a2284d757 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 11 Nov 2026 15:20:14 +0000 Subject: [PATCH 65/70] fix(component): detect render-only lockless changes Compare each component's committed rendered tree before filtering unchanged inputs. Include spec, sidecar, manifest, and no-change-rebuild marker changes so render-only commits enter all-component rebuild selection. Continue to report sources-change only for manifest content changes. Cover output-only changes, added/removed renders, unchanged component trees, and unrelated changes using in-memory Git repositories. Document the additional change-detection input. Validation: mage fix all, mage unit, and mage all build/quality checks passed. Container scenarios were blocked by Docker socket permissions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83cf8df4-30c0-474c-8dca-e3b3a2e54873 --- docs/user/how-to/preview-without-lockfile.md | 8 ++- .../cmds/component/changed_upstreamcommit.go | 41 +++++++++++- .../changed_upstreamcommit_internal_test.go | 67 ++++++++++++++++++- 3 files changed, 109 insertions(+), 7 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 265af112b..9b19cff63 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -74,7 +74,7 @@ when refreshing normally and when checking for stale files with `--check-only`. `component history` skips listed components. `component query` extracts their committed rendered dist-git history and processes it with rpmautospec before querying; uncommitted rendered changes are ignored. `component changed` compares resolved project -inputs for listed components and stored fingerprints for unlisted components; +inputs and committed rendered output for listed components, and stored fingerprints for unlisted components; the current checkout's list determines which comparison to use. ## What Changes @@ -85,7 +85,7 @@ the current checkout's list determines which comparison to use. | Refresh command | `azldev component update` | `azldev component refresh-upstream-commit` | | Inspecting resolved state | `azldev component history`, `azldev component query` | `component query` inspects committed rendered specs; `history` skips lockless components | | Lock consistency checks | On, with `--skip-lock-validation` to opt out | Skipped | -| `component changed` | Compares stored input fingerprints | Compares project configuration resolved at each ref | +| `component changed` | Compares stored input fingerprints | Compares resolved build inputs and committed rendered output at each ref | | Synthetic dist-git history | Derived from lock-file fingerprint changes | Not generated; build and query extract committed rendered history | | Agent skills and MCP tools | Both workflows are available | Both workflows are available | @@ -559,6 +559,10 @@ configuration, upstream commit or local spec-directory contents, overlay source filenames and contents, and the effective distro release version. Documentation, publishing, test-selection, scheduling-hint, snapshot-time, and checkout-path-only fields do not mark a component as changed. +It also compares the committed rendered component directories, including specs, +sidecars, and `.no-change-rebuild` markers. Render-only commits therefore select +the component for rebuilding even when its configuration and upstream pin are +unchanged. `sources-change` still reports only changes to the `sources` manifest. ## Emit Agent Files for the Preview Mode diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit.go b/internal/app/azldev/cmds/component/changed_upstreamcommit.go index 92a3b31b4..0cf141267 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit.go @@ -6,6 +6,7 @@ package component import ( "bytes" "encoding/json" + "errors" "fmt" "log/slog" "path/filepath" @@ -27,9 +28,9 @@ import ( // This file implements 'component changed' for lock-file-free mode. Without lock // files there are no stored fingerprints to compare, so the project configuration -// is loaded independently at both refs and the resolved component build inputs are -// compared directly. The git plumbing helpers are shared with the default -// implementation in changed.go. +// is loaded independently at both refs and the resolved component build inputs and +// rendered trees are compared directly. The git plumbing helpers are shared with +// the default implementation in changed.go. // changedComponentsFromProjectConfigs loads and compares resolved project // components and rendered sources between two git refs. It is the lock-file-free @@ -568,6 +569,22 @@ func buildHistoricalResults( return nil, fmt.Errorf("comparing component %#q:\n%w", name, err) } + if result.ChangeType == changeTypeUnchanged { + fromHash, err := historicalRenderedTreeHash(fromTree, fromProject.renderedSpecsRelDir, name) + if err != nil { + return nil, fmt.Errorf("reading rendered tree for %#q at --from:\n%w", name, err) + } + + toHash, err := historicalRenderedTreeHash(toTree, toProject.renderedSpecsRelDir, name) + if err != nil { + return nil, fmt.Errorf("reading rendered tree for %#q at --to:\n%w", name, err) + } + + if fromHash != toHash { + result.ChangeType = changeTypeChanged + } + } + if result.ChangeType == changeTypeUnchanged { if includeAllComponents && !includeUnchanged { continue @@ -591,6 +608,24 @@ func buildHistoricalResults( return results, nil } +func historicalRenderedTreeHash(tree *object.Tree, renderedDir, name string) (string, error) { + dir, err := components.RenderedSpecDir(renderedDir, name) + if err != nil { + return "", fmt.Errorf("resolving rendered component directory:\n%w", err) + } + + rendered, err := tree.Tree(filepath.ToSlash(dir)) + if errors.Is(err, object.ErrDirectoryNotFound) { + return "", nil + } + + if err != nil { + return "", fmt.Errorf("reading rendered component directory:\n%w", err) + } + + return rendered.Hash.String(), nil +} + // classifyHistoricalComponent compares the serialized build-relevant inputs for a // component at two refs. func classifyHistoricalComponent( diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go index 7c82a2e07..e2cbc4bda 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go @@ -124,13 +124,76 @@ func TestClassifyHistoricalComponent_ExcludedFieldsUnchanged(t *testing.T) { assert.Equal(t, changeTypeUnchanged, result.ChangeType) } -func TestBuildHistoricalResults_UnchangedSkipsSourcesComparison(t *testing.T) { +func TestBuildHistoricalResults_RenderOnlyChanges(t *testing.T) { + for _, testCase := range []struct { + name string + path string + missingTree bool + reverse bool + wantChanged bool + wantSources bool + }{ + {name: "spec", path: "SPECS/c/curl/curl.spec", wantChanged: true}, + {name: "sources", path: "SPECS/c/curl/sources", wantChanged: true, wantSources: true}, + {name: "sidecar", path: "SPECS/c/curl/patches/fix.patch", wantChanged: true}, + {name: "no-change rebuild", path: "SPECS/c/curl/.no-change-rebuild", wantChanged: true}, + {name: "other component", path: "SPECS/z/zlib/zlib.spec"}, + {name: "new render", path: "SPECS/c/curl/curl.spec", missingTree: true, wantChanged: true}, + {name: "removed render", path: "SPECS/c/curl/curl.spec", missingTree: true, reverse: true, wantChanged: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + fromPath := testCase.path + if testCase.missingTree { + fromPath = "README.md" + } + + repo, fromRef, toRef := testRepoWithTwoCommits(t, + map[string][]byte{fromPath: []byte("before")}, + map[string][]byte{testCase.path: []byte("after")}, + ) + + fromTree, err := resolveTree(repo, fromRef) + require.NoError(t, err) + toTree, err := resolveTree(repo, toRef) + require.NoError(t, err) + + if testCase.reverse { + fromTree, toTree = toTree, fromTree + } + + project := &historicalProject{ + comparisonInputs: map[string]componentComparisonInputs{ + "curl": {Config: projectconfig.ComponentConfig{Name: "curl"}}, + }, + renderedSpecsRelDir: "SPECS", + } + + results, err := buildHistoricalResults( + []string{"curl"}, project, project, fromTree, toTree, false, true, + ) + require.NoError(t, err) + + if !testCase.wantChanged { + assert.Empty(t, results) + + return + } + + require.Len(t, results, 1) + assert.Equal(t, changeTypeChanged, results[0].ChangeType) + assert.Equal(t, testCase.wantSources, results[0].SourcesChange) + }) + } +} + +func TestBuildHistoricalResults_UnchangedRenderedTree(t *testing.T) { repo, fromRef, toRef := testRepoWithTwoCommits(t, map[string][]byte{ "SPECS/c/curl/sources": []byte("SHA512 (curl-1.tar.gz) = aaa"), }, map[string][]byte{ - "SPECS/c/curl/sources": []byte("SHA512 (curl-2.tar.gz) = bbb"), + "SPECS/c/curl/sources": []byte("SHA512 (curl-1.tar.gz) = aaa"), + "README.md": []byte("unrelated change"), }, ) From d0364ae55b9e9c8d40b49f7300bb9d9d5d38d126 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 11 Nov 2026 15:20:20 +0000 Subject: [PATCH 66/70] fix(mock): retain stderr from failed rpmautospec probes Include captured stderr when the plugin prerequisite probe fails. Keep the original error wrapped so callers can still inspect its exit status. Add a regression test for a nonzero probe exit that reports diagnostics and prevents the SRPM build from starting. Validation: mage fix all, mage unit, and mage all build/quality checks passed. Container scenarios were blocked by Docker socket permissions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83cf8df4-30c0-474c-8dca-e3b3a2e54873 --- internal/rpm/mock/mock.go | 2 +- internal/rpm/mock/mock_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/internal/rpm/mock/mock.go b/internal/rpm/mock/mock.go index 88bec8dd9..9a787d4d0 100644 --- a/internal/rpm/mock/mock.go +++ b/internal/rpm/mock/mock.go @@ -396,7 +396,7 @@ func (r *Runner) requireRPMAutospec(ctx context.Context) error { } if _, err := cmd.RunAndGetOutput(ctx); err != nil { - return fmt.Errorf("checking mock rpmautospec plugin:\n%w", err) + return fmt.Errorf("checking mock rpmautospec plugin:\n%w\n%s", err, strings.TrimSpace(stderr.String())) } if strings.Contains(stderr.String(), "rpmautospec plugin") && diff --git a/internal/rpm/mock/mock_test.go b/internal/rpm/mock/mock_test.go index 0e0231bf8..9601b84b7 100644 --- a/internal/rpm/mock/mock_test.go +++ b/internal/rpm/mock/mock_test.go @@ -178,6 +178,31 @@ func TestBuildSRPM_MockFails(t *testing.T) { require.ErrorIs(t, err, testError) } +func TestBuildSRPM_RPMAutospecProbeFailure(t *testing.T) { + ctx := newTestCtxWithMockPrereqsPresent() + testError := &exec.ExitError{} + + const diagnostic = "ERROR: could not load mock configuration" + + ctx.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + _, err := io.WriteString(cmd.Stderr, diagnostic+"\n") + require.NoError(t, err) + + return "", testError + } + ctx.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + t.Fatal("must not build when the plugin probe fails") + + return nil + } + + runner := mock.NewRunner(ctx, testMockConfigPath) + err := runner.BuildSRPM(ctx, testSpecPath, testSourceDirPath, testOutputDirPath, + mock.SRPMBuildOptions{RPMAutospec: true}) + require.ErrorIs(t, err, testError) + require.ErrorContains(t, err, diagnostic) +} + func TestBuildSRPM_RPMAutospec(t *testing.T) { ctx := newTestCtxWithMockPrereqsPresent() From 3b575af2a4d14225cfbbef3cfb8d0dcf4f9220ae Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 11 Nov 2026 15:20:27 +0000 Subject: [PATCH 67/70] fix(sources): fetch other branches for pinned upstream commits Do not limit clones to one branch when the component has an effective upstream commit. Explicit pins and locked commits may exist only on other branches and cannot be checked out from a single-branch clone. Keep the single-branch optimization for floating sources. Test generated clone arguments and checkout targets for floating, explicit, and locked pins without invoking external commands. Document pin behavior. Validation: mage fix all, mage unit, and mage all build/quality checks passed. Container scenarios were blocked by Docker socket permissions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83cf8df4-30c0-474c-8dca-e3b3a2e54873 --- docs/user/how-to/preview-without-lockfile.md | 4 ++ .../sourceproviders/fedorasourceprovider.go | 2 +- .../fedorasourceprovider_test.go | 68 +++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 9b19cff63..468e3e673 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -180,6 +180,10 @@ supplies the source type and everything else. Newly written files use one table, upstream-commit = 'abcdef1234567' ``` +Explicit upstream pins may refer to commits on other branches. Source preparation +fetches all branches when an effective commit pin is present; floating lockless +sources retain the single-branch clone optimization. + Existing files with explicit parent tables remain readable. Because a single file may hold a partial component definition for a lockless component, its validation runs after all config files have been merged. Unlisted components retain normal additive diff --git a/internal/providers/sourceproviders/fedorasourceprovider.go b/internal/providers/sourceproviders/fedorasourceprovider.go index 32d35ec08..fd7edd45a 100644 --- a/internal/providers/sourceproviders/fedorasourceprovider.go +++ b/internal/providers/sourceproviders/fedorasourceprovider.go @@ -130,7 +130,7 @@ func (g *FedoraSourcesProviderImpl) GetComponent( // Clone the repository to temp directory with retry for transient network failures. cloneOptions := []git.GitOptions{git.WithGitBranch(g.distroGitBranch)} - if resolved.SingleBranch { + if resolved.SingleBranch && effectiveCommit == "" { cloneOptions = append(cloneOptions, git.WithSingleBranch()) } diff --git a/internal/providers/sourceproviders/fedorasourceprovider_test.go b/internal/providers/sourceproviders/fedorasourceprovider_test.go index a9fa739de..839c7924d 100644 --- a/internal/providers/sourceproviders/fedorasourceprovider_test.go +++ b/internal/providers/sourceproviders/fedorasourceprovider_test.go @@ -6,6 +6,9 @@ package sourceproviders_test import ( "context" "errors" + "os/exec" + "path/filepath" + "slices" "testing" "time" @@ -1104,3 +1107,68 @@ func TestGetComponent_LockedCommitTakesPriorityOverConfigPin(t *testing.T) { err = provider.GetComponent(context.Background(), mockComponent, destDir) require.NoError(t, err) } + +func TestGetComponent_SingleBranchPreservesPinnedCommits(t *testing.T) { + for _, testCase := range []struct { + name string + configPin string + lockedPin string + wantSingle bool + wantCommit string + }{ + {name: "floating", wantSingle: true, wantCommit: "branch-head"}, + {name: "explicit off-branch pin", configPin: testUpstreamCommit, wantCommit: testUpstreamCommit}, + {name: "locked off-branch pin", lockedPin: testUpstreamCommit, wantCommit: testUpstreamCommit}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + gitProvider, err := git.NewGitProviderImpl(env.EventListener, env.CmdFactory) + require.NoError(t, err) + ctrl := gomock.NewController(t) + extractor := fedorasource_test.NewMockFedoraSourceDownloader(ctrl) + provider, err := sourceproviders.NewFedoraSourcesProviderImpl( + env.FS(), env.DryRunnable, gitProvider, extractor, testResolvedDistro(), retry.Disabled()) + require.NoError(t, err) + + config := &projectconfig.ComponentConfig{ + Name: testPackageName, + Spec: projectconfig.SpecSource{UpstreamCommit: testCase.configPin}, + } + if testCase.lockedPin != "" { + config.Locked = &projectconfig.ComponentLockData{UpstreamCommit: testCase.lockedPin} + } + + component := components_testutils.NewMockComponent(ctrl) + component.EXPECT().GetName().AnyTimes().Return(testPackageName) + component.EXPECT().GetConfig().AnyTimes().Return(config) + + cloned, checkedOut := false, false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + if cmd.Args[1] == "clone" { + cloned = true + + assert.Equal(t, testCase.wantSingle, slices.Contains(cmd.Args, "--single-branch")) + cloneDir := cmd.Args[len(cmd.Args)-1] + + return fileutils.WriteFile(env.FS(), filepath.Join(cloneDir, testPackageName+".spec"), + []byte("Name: "+testPackageName), fileperms.PublicFile) + } + + checkedOut = true + + assert.Equal(t, "checkout", cmd.Args[3]) + assert.Equal(t, testCase.wantCommit, cmd.Args[4]) + + return nil + } + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return "branch-head", nil + } + + require.NoError(t, provider.GetComponent(t.Context(), component, destDir, + sourceproviders.WithSingleBranch(), sourceproviders.WithSkipLookaside())) + assert.True(t, cloned) + assert.True(t, checkedOut) + }) + } +} From 516de3b49a58687c19cc9e8870a0305c46b68323 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 11 Nov 2026 15:39:53 +0000 Subject: [PATCH 68/70] fix(component): rely on committed macros for lockless builds Stop forwarding live build defines and with/without flags to Mock when building committed dist-git. The rendered spec already loads the resolved macro file, including TOML undefine semantics. Remove the redundant builder option path for both SRPM and binary builds while retaining local repos, rpmautospec, and no-check behavior. Test that conflicting live macro settings do not reach Mock or overwrite the committed macro file. Document the committed-spec source of macros. Validation: mage fix all, mage check all, and the full unit suite passed. The build and default quality checks passed; Docker-enabled container scenarios are still running separately. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83cf8df4-30c0-474c-8dca-e3b3a2e54873 --- docs/user/how-to/preview-without-lockfile.md | 3 +++ .../content/build-component.md.tmpl | 2 ++ .../withoutlockfile/build-component.md.tmpl | 2 ++ .../azldev/cmds/component/build_lockless.go | 6 +----- .../component/build_lockless_internal_test.go | 19 +++++++++++++++++-- .../core/componentbuilder/componentbuilder.go | 10 +--------- .../componentbuilder/componentbuilder_test.go | 14 +++++++------- 7 files changed, 33 insertions(+), 23 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 468e3e673..b97b0c17b 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -120,6 +120,9 @@ Mock's rpmautospec plugin must be installed on the host as well; enabling an unavailable plugin is an error, even if Mock itself would only warn. The plugin resolves `%autorelease` and `%autochangelog` using the filtered history; no synthetic history, overlay replay, or separate Release bump runs. +Build macros come from the macro file loaded by the committed spec, not duplicate +Mock `--define`, `--with`, or `--without` arguments. Render and commit macro changes +before building; live configuration cannot override committed macro settings. Binary RPMs are built from that SRPM normally. `--no-check` is forwarded through Mock's `--nocheck`; `--srpm-only` skips binary builds. Lockless builds reject `--without-git` and `--rpmdev-bumpspec`. Locked build behavior is unchanged. diff --git a/internal/app/azldev/agentskill/content/build-component.md.tmpl b/internal/app/azldev/agentskill/content/build-component.md.tmpl index 6a1b8f14a..e88476e5d 100644 --- a/internal/app/azldev/agentskill/content/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/build-component.md.tmpl @@ -27,6 +27,8 @@ For lockless components, build uses committed rendered dist-git history extracte with git-filter-repo; render and commit first. It fetches configured sources through the source manager (including download/custom origins), then remaining lookaside payloads, and builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. +Build macros are loaded from the committed spec's macro file, not passed again as +Mock command-line overrides. Render and commit macro changes before building. Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` are rejected for lockless builds; `--no-check` is forwarded normally. Lockless `comp prepare-sources` writes directly to `--output-dir` and downloads/generates diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl index 6153e1264..5c9d21307 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl @@ -27,6 +27,8 @@ For lockless components, build uses committed rendered dist-git history extracte with git-filter-repo; render and commit first. It fetches configured sources through the source manager (including download/custom origins), then remaining lookaside payloads, and builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. +Build macros are loaded from the committed spec's macro file, not passed again as +Mock command-line overrides. Render and commit macro changes before building. Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` are rejected for lockless builds; `--no-check` is forwarded normally. Lockless `comp prepare-sources` writes directly to `--output-dir` and downloads/generates diff --git a/internal/app/azldev/cmds/component/build_lockless.go b/internal/app/azldev/cmds/component/build_lockless.go index f8c065622..1e7ab043a 100644 --- a/internal/app/azldev/cmds/component/build_lockless.go +++ b/internal/app/azldev/cmds/component/build_lockless.go @@ -14,7 +14,6 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/workdir" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" - "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" "github.com/microsoft/azure-linux-dev-tools/internal/utils/defers" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" ) @@ -57,10 +56,7 @@ func buildLocklessComponent( }, &err) builder := componentbuilder.New(env, env.FS(), env, nil, buildEnv, factory) - cfg := component.GetConfig() - builder.UseDistGitSources(dir, mock.CommonBuildOptions{ - With: cfg.Build.With, Without: cfg.Build.Without, Defines: cfg.Build.Defines, - }) + builder.UseDistGitSources(dir) repos := append([]string{}, options.LocalRepoPaths...) if options.LocalRepoWithPublishPath != "" { diff --git a/internal/app/azldev/cmds/component/build_lockless_internal_test.go b/internal/app/azldev/cmds/component/build_lockless_internal_test.go index a0257c483..998da663e 100644 --- a/internal/app/azldev/cmds/component/build_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/build_lockless_internal_test.go @@ -106,7 +106,14 @@ func TestLocklessBuildUsesDistGitAndCleansUp(t *testing.T) { env := testutils.NewTestEnv(t) env.Config.Project.RenderedSpecsDir = "/project/specs" env.Config.LocklessComponents = map[string]bool{"curl": true} - env.Config.Components["curl"] = projectconfig.ComponentConfig{Name: "curl"} + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", + Build: projectconfig.ComponentBuildConfig{ + With: []string{"feature"}, Without: []string{"other"}, + Defines: map[string]string{"dist": ".azl3", "custom": "working-tree-value"}, + Undefines: []string{"dist"}, + }, + } env.CmdFactory.RegisterCommandInSearchPath("git") env.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) @@ -133,7 +140,9 @@ func TestLocklessBuildUsesDistGitAndCleansUp(t *testing.T) { if slices.Contains(cmd.Args, "--subdirectory-filter") { require.Equal(t, "specs/c/curl", cmd.Args[len(cmd.Args)-1]) require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(cloneDir, "curl.spec"), - []byte("committed spec"), fileperms.PublicFile)) + []byte("%{load:%{_sourcedir}/curl.azl.macros}\n"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(cloneDir, "curl.azl.macros"), + []byte("%custom committed-value\n"), fileperms.PublicFile)) } } @@ -149,8 +158,14 @@ func TestLocklessBuildUsesDistGitAndCleansUp(t *testing.T) { built = true + require.NotContains(t, cmd.Args, "--define") + require.NotContains(t, cmd.Args, "--with") + require.NotContains(t, cmd.Args, "--without") require.Contains(t, strings.Join(cmd.Args, " "), "--enable-plugin rpmautospec") require.Equal(t, cloneDir, cmd.Args[slices.Index(cmd.Args, "--sources")+1]) + macros, err := fileutils.ReadFile(env.FS(), filepath.Join(cloneDir, "curl.azl.macros")) + require.NoError(t, err) + require.Equal(t, "%custom committed-value\n", string(macros)) if fail { return sentinel diff --git a/internal/app/azldev/core/componentbuilder/componentbuilder.go b/internal/app/azldev/core/componentbuilder/componentbuilder.go index 57dbd2fc2..956719d9e 100644 --- a/internal/app/azldev/core/componentbuilder/componentbuilder.go +++ b/internal/app/azldev/core/componentbuilder/componentbuilder.go @@ -47,7 +47,6 @@ type Builder struct { buildEnv buildenv.RPMAwareBuildEnv workDirFactory *workdir.Factory distGitDir string - buildOptions mock.CommonBuildOptions } // Ensure that [Builder] implements the [ComponentBuilder] interface. @@ -83,9 +82,8 @@ func (b *Builder) BuildSourcePackage( } // UseDistGitSources builds the prepared repository without replaying overlays or synthetic history. -func (b *Builder) UseDistGitSources(dir string, options mock.CommonBuildOptions) { +func (b *Builder) UseDistGitSources(dir string) { b.distGitDir = dir - b.buildOptions = options } func (b *Builder) BuildBinaryPackage( @@ -175,8 +173,6 @@ func (b *Builder) buildSRPMFromPreparedSources( }, } if b.distGitDir != "" { - srpmOptions.CommonBuildOptions = b.buildOptions - srpmOptions.LocalRepoPaths = localRepoPaths srpmOptions.RPMAutospec = true } @@ -248,10 +244,6 @@ func (b *Builder) buildRPM( NoCheck: noCheck, ForceRebuild: false, // Could be added to config if needed } - if b.distGitDir != "" { - rpmOptions.CommonBuildOptions = b.buildOptions - rpmOptions.LocalRepoPaths = localRepoPaths - } err = buildEnv.BuildRPM(ctx, srpmPath, tempRPMDir, rpmOptions) if err != nil { diff --git a/internal/app/azldev/core/componentbuilder/componentbuilder_test.go b/internal/app/azldev/core/componentbuilder/componentbuilder_test.go index 8111e6f0e..4235e4dc7 100644 --- a/internal/app/azldev/core/componentbuilder/componentbuilder_test.go +++ b/internal/app/azldev/core/componentbuilder/componentbuilder_test.go @@ -23,7 +23,6 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" - "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/stretchr/testify/require" @@ -166,17 +165,17 @@ func TestBuildFromDistGit(t *testing.T) { const dir = "/committed-dist-git" - opts := mock.CommonBuildOptions{With: []string{"feature"}, Defines: map[string]string{"custom": "1"}} // A nil preparer ensures dist-git builds cannot accidentally replay the old preparation pipeline. builder := componentbuilder.New(params.testEnv.Env, params.testEnv.FS(), params.testEnv.Env, nil, params.buildEnv, params.workDirFactory) - builder.UseDistGitSources(dir, opts) + builder.UseDistGitSources(dir) params.buildEnv.EXPECT().BuildSRPM(gomock.Any(), dir+"/"+comp.GetName()+".spec", dir, gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, _, _, output string, options buildenv.SRPMBuildOptions) error { require.True(t, options.RPMAutospec) - require.Equal(t, opts.With, options.With) - require.Equal(t, opts.Defines, options.Defines) + require.Empty(t, options.With) + require.Empty(t, options.Without) + require.Empty(t, options.Defines) require.Equal(t, []string{"/local-repo"}, options.LocalRepoPaths) return fileutils.WriteFile(params.testEnv.FS(), filepath.Join(output, "package.src.rpm"), @@ -187,8 +186,9 @@ func TestBuildFromDistGit(t *testing.T) { params.buildEnv.EXPECT().BuildRPM(gomock.Any(), srpm, gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, _ string, _ string, options buildenv.RPMBuildOptions) error { require.True(t, options.NoCheck) - require.Equal(t, opts.With, options.With) - require.Equal(t, opts.Defines, options.Defines) + require.Empty(t, options.With) + require.Empty(t, options.Without) + require.Empty(t, options.Defines) require.Equal(t, []string{"/local-repo"}, options.LocalRepoPaths) return nil From 3139497272abccea2137520a3c2e2ab6e8499b30 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 11 Nov 2026 16:03:36 +0000 Subject: [PATCH 69/70] feat(component): inline macros in lockless rendered specs Emit resolved build and provenance macros as sorted, lazy %define entries before user overlays. Do not write an .azl.macros file or inject its load directive and Source9999 tag for lockless components. Keep file-based macro loading unchanged for locked components. Cover empty values, deferred expansion, with/without flags, and skipped overlays. Update CLI documentation and emitted agent skills for both workflows. Unit, quality, and changed-line mutation checks pass. Scenario failures for stale global-flag and schema snapshots also reproduce before this change. Network-dependent scenarios were not completed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83cf8df4-30c0-474c-8dca-e3b3a2e54873 --- docs/user/how-to/preview-without-lockfile.md | 36 ++++---- .../cli/azldev_component_prepare-sources.md | 5 +- .../reference/cli/azldev_component_render.md | 4 +- docs/user/reference/config/components.md | 9 +- .../app/azldev/agentskill/agentskill_test.go | 2 +- .../azldev/agentskill/content/azldev.md.tmpl | 8 +- .../content/build-component.md.tmpl | 8 +- .../agentskill/content/comp-toml.md.tmpl | 6 +- .../content/refresh-upstream-commit.md.tmpl | 4 +- .../content/withoutlockfile/azldev.md.tmpl | 8 +- .../withoutlockfile/build-component.md.tmpl | 8 +- .../azldev/cmds/component/preparesources.go | 5 +- .../preparesources_lockless_internal_test.go | 19 +++- internal/app/azldev/cmds/component/render.go | 4 +- .../render_lockless_internal_test.go | 9 +- .../app/azldev/core/sources/sourceprep.go | 90 +++++++++++++------ .../azldev/core/sources/sourceprep_test.go | 81 ++++++++++++++++- 17 files changed, 224 insertions(+), 82 deletions(-) diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index b97b0c17b..dd614a06c 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -111,8 +111,8 @@ The temporary query repositories are removed on success and failure. Locked queries continue to read working-tree rendered files, including uncommitted edits. `component build` fetches configured source files through the normal source manager, -including custom generation and download-origin fallback unless `disable-origins` -is enabled. It then downloads any remaining payloads listed in the extracted +including custom generation. Download-origin fallback is enabled unless +`disable-origins` is set. It then downloads any remaining payloads listed in the extracted `sources` manifest from lookaside and builds the SRPM with Mock's `rpmautospec` plugin enabled and the extracted repository passed as `--sources`. This requires Mock 5.3 or newer and rpmautospec available from the build distro's package repositories. @@ -120,9 +120,13 @@ Mock's rpmautospec plugin must be installed on the host as well; enabling an unavailable plugin is an error, even if Mock itself would only warn. The plugin resolves `%autorelease` and `%autochangelog` using the filtered history; no synthetic history, overlay replay, or separate Release bump runs. -Build macros come from the macro file loaded by the committed spec, not duplicate +Build macros come from the committed spec, not duplicate Mock `--define`, `--with`, or `--without` arguments. Render and commit macro changes before building; live configuration cannot override committed macro settings. +Lockless rendering emits sorted `%define` entries at the top of the spec instead +of a separate `.azl.macros` file, `%{load:...}` directive, or generated `Source9999`. +These lazy definitions preserve references to macros defined later in the spec. +Locked components retain the separate macros file and its source entry. Binary RPMs are built from that SRPM normally. `--no-check` is forwarded through Mock's `--nocheck`; `--srpm-only` skips binary builds. Lockless builds reject `--without-git` and `--rpmdev-bumpspec`. Locked build behavior is unchanged. @@ -138,8 +142,8 @@ Nonempty output requires `--force`, which removes existing contents before preparation. Failures can leave partially prepared output. It fetches upstream lookaside payloads with the spec/sidecars, prepares the upstream release/changelog, then downloads/generates configured `source-files`. Like locked preparation, it -acquires payloads before writing the macro file and applying overlays. Archive -overlays run first, followed by macro-file loading, configured spec/loose-file +acquires payloads before emitting build macros and applying overlays. Archive +overlays run first, followed by inline macro definitions, configured spec/loose-file overlays, and generated check-skip/header overlays. It updates `sources` metadata with the configured post-overlay hashes. It does not compare rendered output, apply a final render release bump, or commit changes. @@ -155,7 +159,7 @@ The source-preparation flags also apply to lockless components: | Flag | Behavior | |------|----------| | `--without-git` | Remove `.git` from prepared output after successful preparation. It remains available for upstream release/changelog preparation; no synthetic history is generated with or without the flag. | -| `--skip-overlays` | Skip configured and generated overlays, macro-file creation/loading, and `sources` metadata updates. Source acquisition still runs unless `--skip-sources` is also set. Upstream release/changelog preparation still runs. | +| `--skip-overlays` | Skip configured and generated overlays, inline macro definitions, and `sources` metadata updates. Source acquisition still runs unless `--skip-sources` is also set. Upstream release/changelog preparation still runs. | | `--skip-sources` | Skip upstream lookaside downloads, configured source downloads/generation, and archive overlays. Spec/sidecars are still fetched; other overlays and `sources` metadata updates still run unless `--skip-overlays` is set. Overlays requiring missing payloads can fail. | | `--allow-no-hashes` | Compute missing hashes from acquired files when updating `sources`, defaulting to SHA-512 when `hash-type` is omitted. This does not edit TOML. With `--skip-sources`, missing hashes cannot be computed and cause an error; with `--skip-overlays`, metadata updates and hash computation are skipped. | @@ -255,28 +259,28 @@ check: carryover and upstream-change comparison still use committed history, and failure recovery can discard uncommitted files in the component output directory. Locked components do not trigger this check. -For upstream components, the renderer clones only the branch selected by the -component's effective -`spec.upstream-distro` (or the project default) into temporary storage. The -resolved distro version's `dist-git-branch` supplies the branch name. The clone -retains that branch's full history, not every upstream branch, so pinned commits -and snapshot selection remain available without a shallow-history limit. -This branch restriction is specific to lockless preparation. Locked components -retain normal multi-branch cloning so existing pins on other branches remain usable. +For unpinned upstream components, the renderer clones only the branch selected by +the component's effective `spec.upstream-distro` (or the project default) into +temporary storage. The resolved distro version's `dist-git-branch` supplies the +branch name. The clone retains that branch's full history so snapshot selection +remains available without a shallow-history limit. Pinned components fetch all +branches so commits outside the selected branch remain usable. This branch +restriction is specific to unpinned lockless preparation; locked components +retain normal multi-branch cloning. Immediately after checking out the pinned upstream commit, rendering captures optional Fedora provenance values for `build.emit-upstream-provenance`, before release/changelog carryover or bumping changes the spec. If the upstream release uses `%autorelease` and `build.defines` does not already provide `fedora_upstream_release`, it runs host `rpmautospec calculate-release` at this -point. The captured values are reused later when writing the macros file, without +point. The captured values are reused later when emitting inline macros, without repeating the query against the modified checkout. No provenance query runs when provenance is disabled or the component is not sourced from Fedora upstream. The renderer then finds the existing committed output under `//`, ignoring uncommitted spec changes. It prepares the changelog first, then Release, before -writing the macros file, applying overlays, or updating `sources`. +emitting inline macros, applying overlays, or updating `sources`. Changelog handling is independent of version changes. For non-`%autorelease` specs, the committed inline `%changelog` section is preserved. For an initial diff --git a/docs/user/reference/cli/azldev_component_prepare-sources.md b/docs/user/reference/cli/azldev_component_prepare-sources.md index 816398e8e..952325a08 100644 --- a/docs/user/reference/cli/azldev_component_prepare-sources.md +++ b/docs/user/reference/cli/azldev_component_prepare-sources.md @@ -19,8 +19,9 @@ and preserves fetched upstream Git metadata. Nonempty output requires '--force'. Current TOML and file inputs use the working tree, including modified and untracked files. Release/changelog baselines use only committed component dist-git files; uncommitted edits in that existing dist-git directory are ignored. -It downloads/generates source payloads before macro-file setup and configured -overlays. Archive overlays run before spec and loose-file overlays. It does not +It downloads/generates source payloads before inline macro definitions and configured +overlays. Lockless specs contain %define entries, not a separate macros file, +load directive, or Source9999 entry. Archive overlays run before spec and loose-file overlays. It does not compare rendered output, apply a final render release bump, or commit changes. Render history comes from 'project.rendered-specs-dir'//, independently of '--output-dir'. No synthetic history is generated. diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 2e719f75a..c17557e01 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -47,8 +47,8 @@ Lockless rendering never uses mock or rpmautospec process-distgit. The spec retains its RPM macros, including %autorelease and %autochangelog. All prepared files except .git are retained, without filtering or spectool. Host rpmautospec generate-changelog/calculate-release are used only when needed for -release history or provenance. Provenance is captured before carryover/bumping -and reused for the macros file. Locked components keep their existing behavior. +release history or provenance. Provenance is captured before carryover/bumping and inlined with %define. +Lockless output adds no macros file, load directive, or Source9999. Locked components keep their existing behavior. Local lockless components copy local files except the TOML defining the spec path. Their Release and changelog are never automatically modified, even with --rpmdev-bumpspec. diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 70abc53b8..5e1324622 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -259,10 +259,11 @@ emit-upstream-provenance = true The values are captured immediately after fetching the pinned upstream checkout, **before** release/changelog carryover, release bumping, or overlays can modify -the spec. The captured values are reused when writing the component's generated -macros file (loaded via `%{load:...}`), rather than querying the modified checkout. +the spec. The captured values are reused when emitting inline `%define` entries +for lockless components, or writing the generated macros file (loaded via +`%{load:...}`) for locked components, rather than querying the modified checkout. -For specs whose `Release` uses rpmautospec (`Release: %autorelease`), the pristine Fedora release number is computed by running `rpmautospec calculate-release` against the upstream dist-git checkout (whose history is Fedora's, not azldev's synthetic overlay history). Lockless rendering runs this query on the host; other workflows use the project distro's mock chroot. The query runs only when overlays will be applied, provenance is enabled for a Fedora upstream component, and `build.defines` does not already supply `fedora_upstream_release`. Static releases do not need this query. If resolution is unavailable or fails, `%fedora_upstream_release` is skipped (with a warning) rather than emitting the literal `%autorelease`; the query is not retried when writing the macros file. +For specs whose `Release` uses rpmautospec (`Release: %autorelease`), the pristine Fedora release number is computed by running `rpmautospec calculate-release` against the upstream dist-git checkout (whose history is Fedora's, not azldev's synthetic overlay history). Lockless rendering runs this query on the host; other workflows use the project distro's mock chroot. The query runs only when overlays will be applied, provenance is enabled for a Fedora upstream component, and `build.defines` does not already supply `fedora_upstream_release`. Static releases do not need this query. If resolution is unavailable or fails, `%fedora_upstream_release` is skipped (with a warning) rather than emitting the literal `%autorelease`; the query is not retried when emitting macros. Example: for a component pinned to Fedora 43's `grub2-2.12-5.fc43`, the spec can reference: @@ -276,7 +277,7 @@ Notes: - The flag has no effect on local or SRPM components (they have no upstream provenance) or on non-Fedora upstreams; for those it is silently ignored. - If a component defines a macro of the same name via `build.defines`, the user-defined value wins — the injected value does not overwrite it. -- **How tag values are resolved:** `Version` and `Release` are read as plain text from the pristine upstream spec. `%{?dist}` is substituted with the Fedora dist tag, and `%autorelease` is resolved to a concrete number via `rpmautospec` (see above). Any *other* in-spec macros — e.g. `Version: %{majorver}.%{minorver}` — are emitted into the macros file unexpanded. Because that file is loaded back into the same spec via `%{load:...}`, RPM expands them lazily at build time using the spec's own macro definitions, so the consuming spec still sees the correct fully-expanded value. +- **How tag values are resolved:** `Version` and `Release` are read as plain text from the pristine upstream spec. `%{?dist}` is substituted with the Fedora dist tag, and `%autorelease` is resolved to a concrete number via `rpmautospec` (see above). Any *other* in-spec macros — e.g. `Version: %{majorver}.%{minorver}` — remain unexpanded in the generated definitions. Lockless specs use lazy `%define` entries; locked specs load their macros file via `%{load:...}`. Both expand these references at build time using the spec's own macro definitions, so the consuming spec still sees the correct fully-expanded value. Limitations: diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index cef39a2c8..fe912b439 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -296,7 +296,7 @@ func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { assert.Contains(t, doc, "All prepared dist-git files except `.git` are retained") assert.Contains(t, doc, "immediately after checking out the pinned upstream commit") assert.Contains(t, doc, "that step never repeats the provenance query") - assert.Contains(t, doc, "the changelog before Release, then writes the macros file") + assert.Contains(t, doc, "the changelog before Release, then emits inline macros") assert.Contains(t, doc, "Version detection and carryover use the upstream spec before overlays") assert.Contains(t, doc, "Overlays can override the prepared release and changelog") assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index f775de02f..db0cae5b9 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -147,15 +147,17 @@ using render-style preparation. Current TOML and file inputs use the working tre including modified and untracked files. Release/changelog baselines use only committed component dist-git files; edits in the existing dist-git directory are ignored. Nonempty output requires `--force`; failures can -leave partial output. It downloads/generates sources before macro-file setup and -overlays, matching locked preparation. Archive overlays run before spec and +leave partial output. It downloads/generates sources before inline macro setup and +overlays. Lockless specs use `%define` entries, with no separate macros file, +load directive, or generated `Source9999`; locked specs retain their macros file. +Archive overlays run before spec and loose-file overlays. It does not compare rendered output, apply a final render release bump, or commit. It reads render history from `project.rendered-specs-dir` using `/`, never from `--output-dir`. It never generates synthetic history. `--without-git` removes `.git` after successful preparation, preserving it on failure. It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. -Skipping overlays also skips macro-file setup and `sources` metadata updates; +Skipping overlays also skips inline macro setup and `sources` metadata updates; skipping sources also skips archive overlays. Missing hashes cannot be computed when source acquisition is skipped. Use `comp dist-git-repo` for committed history extraction. diff --git a/internal/app/azldev/agentskill/content/build-component.md.tmpl b/internal/app/azldev/agentskill/content/build-component.md.tmpl index e88476e5d..4882f78ff 100644 --- a/internal/app/azldev/agentskill/content/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/build-component.md.tmpl @@ -27,12 +27,14 @@ For lockless components, build uses committed rendered dist-git history extracte with git-filter-repo; render and commit first. It fetches configured sources through the source manager (including download/custom origins), then remaining lookaside payloads, and builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. -Build macros are loaded from the committed spec's macro file, not passed again as -Mock command-line overrides. Render and commit macro changes before building. +Build macros are inline `%define` entries in the committed spec, not passed again +as Mock command-line overrides. Lockless rendering adds no macros file, load +directive, or `Source9999` entry. Locked components retain their macros file. +Render and commit macro changes before building. Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` are rejected for lockless builds; `--no-check` is forwarded normally. Lockless `comp prepare-sources` writes directly to `--output-dir` and downloads/generates -sources before macro-file setup, then applies archive overlays before spec and +sources before inline macro setup, then applies archive overlays before spec and loose-file overlays. History comes from `project.rendered-specs-dir`, not `--output-dir`; no synthetic history is generated. `--without-git` removes `.git` after successful preparation, preserving it on failure. diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index cfb66916d..214cd0077 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -94,8 +94,10 @@ synthetic history still determine the number and order of bumps. Without the flag, no bumpspec runtime discovery occurs and the legacy static calculation is used for locked components. Lockless builds reject this flag and use Mock's rpmautospec plugin with committed dist-git history; lockless `comp prepare-sources` -writes directly to `--output-dir` and downloads/generates sources before macro-file -setup, then applies archive overlays before spec/loose-file overlays. Nonempty output +writes directly to `--output-dir` and downloads/generates sources before inline macro +setup, then applies archive overlays before spec/loose-file overlays. Lockless specs +use `%define` entries without a macros file, load directive, or generated `Source9999`. +Locked specs retain their separate macros file. Nonempty output requires `--force`; failures can leave partial output. It does not commit changes. It reads history from `project.rendered-specs-dir`, not `--output-dir`, without generating synthetic history. `--without-git` removes `.git` diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl index 0403d3653..88d520014 100644 --- a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -89,10 +89,10 @@ immediately after checking out the pinned upstream commit, before release/changelog carryover or bumping. Host `rpmautospec calculate-release` runs here only for an upstream `%autorelease` whose `fedora_upstream_release` is not already supplied by `build.defines`. The captured values are reused -when writing the macros file; that step never repeats the provenance query. +when emitting inline `%define` entries; that step never repeats the provenance query. After reading render history and collecting upstream changes, rendering prepares -the changelog before Release, then writes the macros file, applies overlays, and +the changelog before Release, then emits inline macros, applies overlays, and updates `sources` before comparison and the final `rpmdev-bumpspec`. Release edits preserve the existing spaces and tabs after `Release:`. Version detection and carryover use the upstream spec before overlays. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index 0bffaefd2..67dbbfccd 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -147,15 +147,17 @@ using render-style preparation. Current TOML and file inputs use the working tre including modified and untracked files. Release/changelog baselines use only committed component dist-git files; edits in the existing dist-git directory are ignored. Nonempty output requires `--force`; failures can -leave partial output. It downloads/generates sources before macro-file setup and -overlays, matching locked preparation. Archive overlays run before spec and +leave partial output. It downloads/generates sources before inline macro setup and +overlays. Lockless specs use `%define` entries, with no separate macros file, +load directive, or generated `Source9999`; locked specs retain their macros file. +Archive overlays run before spec and loose-file overlays. It does not compare rendered output, apply a final render release bump, or commit. It reads render history from `project.rendered-specs-dir` using `/`, never from `--output-dir`. It never generates synthetic history. `--without-git` removes `.git` after successful preparation, preserving it on failure. It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. -Skipping overlays also skips macro-file setup and `sources` metadata updates; +Skipping overlays also skips inline macro setup and `sources` metadata updates; skipping sources also skips archive overlays. Missing hashes cannot be computed when source acquisition is skipped. Use `comp dist-git-repo` for committed history extraction. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl index 5c9d21307..436275a42 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl @@ -27,12 +27,14 @@ For lockless components, build uses committed rendered dist-git history extracte with git-filter-repo; render and commit first. It fetches configured sources through the source manager (including download/custom origins), then remaining lookaside payloads, and builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. -Build macros are loaded from the committed spec's macro file, not passed again as -Mock command-line overrides. Render and commit macro changes before building. +Build macros are inline `%define` entries in the committed spec, not passed again +as Mock command-line overrides. Lockless rendering adds no macros file, load +directive, or `Source9999` entry. Locked components retain their macros file. +Render and commit macro changes before building. Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` are rejected for lockless builds; `--no-check` is forwarded normally. Lockless `comp prepare-sources` writes directly to `--output-dir` and downloads/generates -sources before macro-file setup, then applies archive overlays before spec and +sources before inline macro setup, then applies archive overlays before spec and loose-file overlays. History comes from `project.rendered-specs-dir`, not `--output-dir`; no synthetic history is generated. `--without-git` removes `.git` after successful preparation, preserving it on failure. diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index 61c2e129b..8fc598319 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -55,8 +55,9 @@ and preserves fetched upstream Git metadata. Nonempty output requires '--force'. Current TOML and file inputs use the working tree, including modified and untracked files. Release/changelog baselines use only committed component dist-git files; uncommitted edits in that existing dist-git directory are ignored. -It downloads/generates source payloads before macro-file setup and configured -overlays. Archive overlays run before spec and loose-file overlays. It does not +It downloads/generates source payloads before inline macro definitions and configured +overlays. Lockless specs contain %define entries, not a separate macros file, +load directive, or Source9999 entry. Archive overlays run before spec and loose-file overlays. It does not compare rendered output, apply a final render release bump, or commit changes. Render history comes from 'project.rendered-specs-dir'//, independently of '--output-dir'. No synthetic history is generated. diff --git a/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go b/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go index dbd9ccf3b..e7a6aafad 100644 --- a/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go @@ -412,9 +412,11 @@ func TestLocklessPrepareSourcesUsesWorkingInputs(t *testing.T) { require.NoError(t, err) assert.Equal(t, "working overlay input", string(copied)) - macros, err := fileutils.ReadFile(env.FS(), "/external/output/curl.azl.macros") + assert.Contains(t, string(prepared), "%define working_input working\n") + + exists, err := fileutils.Exists(env.FS(), "/external/output/curl.azl.macros") require.NoError(t, err) - assert.Contains(t, string(macros), "%working_input working\n") + assert.False(t, exists) }) } } @@ -558,18 +560,22 @@ func TestLocklessPrepareSourcesFlags(t *testing.T) { assert.Equal(t, !testCase.options.SkipSources, exists) exists, err = fileutils.Exists(env.FS(), filepath.Join(dir, "curl.azl.macros")) require.NoError(t, err) - assert.Equal(t, !testCase.options.SkipOverlays, exists) + assert.False(t, exists) content, err := fileutils.ReadFile(env.FS(), filepath.Join(dir, "sources")) require.NoError(t, err) specContent, err := fileutils.ReadFile(env.FS(), filepath.Join(dir, "curl.spec")) require.NoError(t, err) + assert.NotContains(t, string(specContent), "%{load:") + assert.NotContains(t, string(specContent), "Source9999") + if testCase.options.SkipOverlays { assert.Equal(t, "# upstream metadata\n", string(content)) assert.Equal(t, renderTestSpec("1", "7", ""), string(specContent)) } else { assert.Contains(t, string(specContent), "Summary: Prepared summary\n") + assert.Contains(t, string(specContent), "%define prepare_fixture ready\n") hash, hashType := helloSHA256, fileutils.HashTypeSHA256 if testCase.missingType { @@ -729,7 +735,12 @@ func assertLocklessPreparedSourceTree(t *testing.T, fs opctx.FS, dir, release, h specContent, err := fileutils.ReadFile(fs, filepath.Join(dir, "curl.spec")) require.NoError(t, err) assert.Contains(t, string(specContent), "Summary: Prepared summary\n") - assert.Equal(t, 1, strings.Count(string(specContent), "%{load:")) + assert.NotContains(t, string(specContent), "%{load:") + assert.NotContains(t, string(specContent), "Source9999") + + exists, err := fileutils.Exists(fs, filepath.Join(dir, "curl.azl.macros")) + require.NoError(t, err) + assert.False(t, exists) assert.Contains(t, string(specContent), "Release: "+release+"\n") content, err := fileutils.ReadFile(fs, filepath.Join(dir, "generated-input.txt")) diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 72a8554cc..ed293bff7 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -119,8 +119,8 @@ Lockless rendering never uses mock or rpmautospec process-distgit. The spec retains its RPM macros, including %autorelease and %autochangelog. All prepared files except .git are retained, without filtering or spectool. Host rpmautospec generate-changelog/calculate-release are used only when needed for -release history or provenance. Provenance is captured before carryover/bumping -and reused for the macros file. Locked components keep their existing behavior.` + "\n\n" + renderLocalHelp + ` +release history or provenance. Provenance is captured before carryover/bumping and inlined with %define. +Lockless output adds no macros file, load directive, or Source9999. Locked components keep their existing behavior.` + "\n\n" + renderLocalHelp + ` When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go index d442ef747..f9b6bb730 100644 --- a/internal/app/azldev/cmds/component/render_lockless_internal_test.go +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -1433,12 +1433,13 @@ upstream-commit = "aaaaaaa" assert.Contains(t, string(content), releaseTag+wantRelease+"\n") assert.Contains(t, string(content), "Version: "+overlayVersion+"\n") assert.Contains(t, string(content), "%changelog\n* Overlaid entry\n") - assert.Contains(t, string(content), "%{load:%{_sourcedir}/curl.azl.macros}") - assert.Contains(t, string(content), "Source9999") + assert.Contains(t, string(content), "%define _with_feature 1\n") + assert.NotContains(t, string(content), "%{load:") + assert.NotContains(t, string(content), "Source9999") - macros, err := fileutils.ReadFile(env.TestFS, "/staging/curl/curl.azl.macros") + exists, err := fileutils.Exists(env.TestFS, "/staging/curl/curl.azl.macros") require.NoError(t, err) - assert.Contains(t, string(macros), "%_with_feature 1\n") + assert.False(t, exists) manifest, err := fileutils.ReadFile(env.TestFS, "/staging/curl/sources") require.NoError(t, err) diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 9a17818b1..4cdeff6f9 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -437,39 +437,56 @@ func (p *sourcePreparerImpl) validateArchiveOverlayConfig(component components.C return nil } -// applyOverlaysToSources writes the macros file and then applies all overlays. +// applyOverlaysToSources prepares macro definitions and then applies all overlays. // It returns the names of any archives that were repacked by archive overlays // (empty in dry-run mode or when no archive overlays ran), so the caller can // rehash exactly those entries in the 'sources' file. func (p *sourcePreparerImpl) applyOverlaysToSources( component components.Component, outputDir string, macros map[string]string, ) ([]string, error) { - var macrosFileName string + macroOverlays, err := p.prepareMacroOverlays(component, outputDir, macros) + if err != nil { + return nil, err + } - macrosFilePath, err := p.writeMacrosFile(component, outputDir, macros) + repackedArchives, err := p.applyOverlays(component, outputDir, macroOverlays) if err != nil { - return nil, fmt.Errorf("failed to write macros file for component %#q:\n%w", + return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } - if macrosFilePath != "" { - macrosFileName = filepath.Base(macrosFilePath) + return repackedArchives, nil +} + +func (p *sourcePreparerImpl) prepareMacroOverlays( + component components.Component, outputDir string, macros map[string]string, +) ([]projectconfig.ComponentOverlay, error) { + if p.withoutLockfile { + return synthesizeInlineMacroOverlays(macros), nil } - repackedArchives, err := p.applyOverlays(component, outputDir, macrosFileName) + macrosFilePath, err := p.writeMacrosFile(component, outputDir, macros) if err != nil { - return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", - component.GetName(), err) + return nil, fmt.Errorf("failed to write macros file for component %#q:\n%w", component.GetName(), err) } - return repackedArchives, nil + if macrosFilePath == "" { + return nil, nil + } + + overlays, err := synthesizeMacroLoadOverlays(filepath.Base(macrosFilePath)) + if err != nil { + return nil, fmt.Errorf("failed to compute macros load overlays:\n%w", err) + } + + return overlays, nil } // applyOverlays applies all overlays (user-defined and system-generated) to the // component sources. It returns the names of any archives that were repacked by // archive overlays. func (p *sourcePreparerImpl) applyOverlays( - component components.Component, sourcesDirPath, macrosFileName string, + component components.Component, sourcesDirPath string, macroOverlays []projectconfig.ComponentOverlay, ) ([]string, error) { event := p.eventListener.StartEvent("Applying overlays", "component", component.GetName()) defer event.End() @@ -479,10 +496,7 @@ func (p *sourcePreparerImpl) applyOverlays( return nil, err } - allOverlays, err := p.collectOverlays(component, macrosFileName) - if err != nil { - return nil, fmt.Errorf("failed to collect overlays for component %#q:\n%w", component.GetName(), err) - } + allOverlays := p.collectOverlays(component, macroOverlays) if len(allOverlays) == 0 { return nil, nil @@ -541,28 +555,20 @@ func (p *sourcePreparerImpl) applyArchiveOverlayGroup( } // collectOverlays gathers all overlays for a component into a single ordered slice: -// macros-load first, then user overlays, followed by check-skip and file-header overlays. +// macro definitions first, then user overlays, followed by check-skip and file-header overlays. func (p *sourcePreparerImpl) collectOverlays( - component components.Component, macrosFileName string, -) ([]projectconfig.ComponentOverlay, error) { + component components.Component, macroOverlays []projectconfig.ComponentOverlay, +) []projectconfig.ComponentOverlay { config := component.GetConfig() var allOverlays []projectconfig.ComponentOverlay - if macrosFileName != "" { - macroOverlays, err := synthesizeMacroLoadOverlays(macrosFileName) - if err != nil { - return nil, fmt.Errorf("failed to compute macros load overlays:\n%w", err) - } - - allOverlays = append(allOverlays, macroOverlays...) - } - + allOverlays = append(allOverlays, macroOverlays...) allOverlays = append(allOverlays, config.Overlays...) allOverlays = append(allOverlays, synthesizeCheckSkipOverlays(config.Build.Check)...) allOverlays = append(allOverlays, generateFileHeaderOverlay()...) - return allOverlays, nil + return allOverlays } // initSourcesRepo initializes a new git repository in sourcesDirPath, stages all files, @@ -1441,6 +1447,34 @@ func renderMacrosFile(macros map[string]string) string { return strings.Join(lines, "\n") + "\n" } +func synthesizeInlineMacroOverlays(macros map[string]string) []projectconfig.ComponentOverlay { + if len(macros) == 0 { + return nil + } + + names := lo.Keys(macros) + slices.Sort(names) + + lines := []string{"# Build macros generated by azldev."} + + for _, name := range names { + value := macros[name] + if value == "" { + value = "%{nil}" + } + + // Lazy definitions retain macros-file semantics, including references to later spec tags. + lines = append(lines, fmt.Sprintf("%%define %s %s", name, value)) + } + + lines = append(lines, "") + + return []projectconfig.ComponentOverlay{{ + Type: projectconfig.ComponentOverlayPrependSpecLines, + Lines: lines, + }} +} + func synthesizeMacroLoadOverlays(macrosFileName string) ([]projectconfig.ComponentOverlay, error) { // Basic check that the macros file name is valid and doesn't require escaping. if strings.ContainsFunc(macrosFileName, func(r rune) bool { diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index eaae4a221..07139b171 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -496,8 +496,87 @@ func TestPrepareSources_WritesMacrosFile(t *testing.T) { assert.Contains(t, specStr, "Source9999") } -// Tests for GenerateMacrosFileContents - these test content generation in isolation. +func TestPrepareSources_InlinesLocklessMacros(t *testing.T) { + for _, testCase := range []struct { + name string + skipOverlays bool + removeAll bool + }{ + {name: "configured macros"}, + {name: "all macros undefined", removeAll: true}, + {name: "overlays skipped", skipOverlays: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + ctx := testctx.NewCtx() + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + manager := sourceproviders_test.NewMockSourceManager(ctrl) + + config := projectconfig.ComponentConfig{ + Build: projectconfig.ComponentBuildConfig{ + With: []string{"feature", "removed"}, Without: []string{"legacy"}, + Defines: map[string]string{ + "empty": "", "late": "%{version}", "words": "two words", + }, + Undefines: []string{"_with_removed"}, + }, + } + if testCase.removeAll { + config.Build.Undefines = append(config.Build.Undefines, + "_with_feature", "_without_legacy", "empty", "late", "words") + } + + component.EXPECT().GetName().AnyTimes().Return("my-package") + component.EXPECT().GetConfig().AnyTimes().Return(&config) + manager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(nil) + specPath := filepath.Join(testOutputDir, "my-package.spec") + manager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir, gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, _ string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile(ctx.FS(), specPath, []byte("Name: my-package\nVersion: 1\n"), + fileperms.PublicFile) + }) + + preparer, err := sources.NewPreparer(manager, ctx.FS(), ctx, ctx, sources.WithoutLockfileHistory()) + require.NoError(t, err) + require.NoError(t, preparer.PrepareSources(ctx, component, testOutputDir, !testCase.skipOverlays)) + content, err := fileutils.ReadFile(ctx.FS(), specPath) + require.NoError(t, err) + specText := string(content) + assert.NotContains(t, specText, "%{load:") + assert.NotContains(t, specText, "Source9999") + + exists, err := fileutils.Exists(ctx.FS(), filepath.Join(testOutputDir, "my-package"+sources.MacrosFileExtension)) + require.NoError(t, err) + assert.False(t, exists) + + if testCase.removeAll || testCase.skipOverlays { + assert.NotContains(t, specText, "%define ") + + return + } + + last := -1 + + for _, line := range []string{ + "%define _with_feature 1\n", + "%define _without_legacy 1\n", + "%define empty %{nil}\n", + "%define late %{version}\n", + "%define words two words\n", + "Name: my-package\n", + } { + index := strings.Index(specText, line) + assert.Greater(t, index, last, "expected ordered inline definition %q", line) + last = index + } + + assert.NotContains(t, specText, "_with_removed") + }) + } +} + +// Tests for GenerateMacrosFileContents - these test content generation in isolation. func TestGenerateMacrosFileContents_EmptyConfig(t *testing.T) { contents := sources.GenerateMacrosFileContents(projectconfig.ComponentBuildConfig{}) From 922e31f9b3afa34d569009d5ccdee56421333d0b Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 11 Nov 2026 16:16:19 +0000 Subject: [PATCH 70/70] test(scenario): refresh lockless CLI snapshots Regenerate local and container help, MCP, and schema snapshots to match removal of the global --without-lockfile flag and the updated lockless file-filter description. Verify with mage scenarioUpdate and mage scenario, both using TESTING_DISABLE_NETWORK_TESTS=1. The full network-dependent suite remains unverified after its earlier stall during Mock chroot cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83cf8df4-30c0-474c-8dca-e3b3a2e54873 --- .../TestMCPServerMode_1.snap.json | 55 ------------------- ...pshotsContainer_--bogus-flag_stderr_1.snap | 1 - ...estSnapshotsContainer_--help_stdout_1.snap | 1 - ...ainer_config_generate-schema_stdout_1.snap | 2 +- .../TestSnapshotsContainer_help_stdout_1.snap | 1 - .../TestSnapshots_--bogus-flag_stderr_1.snap | 1 - .../TestSnapshots_--help_stdout_1.snap | 1 - ...tSnapshots_--help_with_color_stdout_1.snap | 1 - ...shots_config_generate-schema_stdout_1.snap | 2 +- .../TestSnapshots_help_stdout_1.snap | 1 - 10 files changed, 2 insertions(+), 64 deletions(-) diff --git a/scenario/__snapshots__/TestMCPServerMode_1.snap.json b/scenario/__snapshots__/TestMCPServerMode_1.snap.json index 663b83ff7..5119f0cfd 100755 --- a/scenario/__snapshots__/TestMCPServerMode_1.snap.json +++ b/scenario/__snapshots__/TestMCPServerMode_1.snap.json @@ -105,11 +105,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [ @@ -216,11 +211,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -325,11 +315,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -424,11 +409,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -543,11 +523,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -625,11 +600,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -703,11 +673,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -786,11 +751,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -879,11 +839,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [ @@ -959,11 +914,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -1056,11 +1006,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], diff --git a/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap index 6832b09a4..48c9a1b58 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap @@ -31,7 +31,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap index e15aef19c..0b77a7417 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap @@ -38,7 +38,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 9cf47bd35..795c1ea39 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -345,7 +345,7 @@ "skip-file-filter": { "type": "boolean", "title": "Skip file filter", - "description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags" + "description": "Disable post-render file filtering for locked components with unexpandable macros in Source/Patch tags. Lockless rendering always preserves all prepared files except .git." } }, "additionalProperties": false, diff --git a/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap index e15aef19c..0b77a7417 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap @@ -38,7 +38,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap b/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap index 6832b09a4..48c9a1b58 100755 --- a/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap +++ b/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap @@ -31,7 +31,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap index e15aef19c..0b77a7417 100755 --- a/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap @@ -38,7 +38,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap index 4a47c31e2..3dfbd2bb4 100755 --- a/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap @@ -38,7 +38,6 @@ Meta commands: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 9cf47bd35..795c1ea39 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -345,7 +345,7 @@ "skip-file-filter": { "type": "boolean", "title": "Skip file filter", - "description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags" + "description": "Disable post-render file filtering for locked components with unexpandable macros in Source/Patch tags. Lockless rendering always preserves all prepared files except .git." } }, "additionalProperties": false, diff --git a/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap index e15aef19c..0b77a7417 100755 --- a/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap @@ -38,7 +38,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget).